Skip to content

An AI-powered search engine with a generative UI

fix(streaming): cap replayed historical answer text (#996)

Yoshiki Miura244336f2cf15
diff --git a/lib/streaming/helpers/__tests__/cap-historical-answer-text.test.ts b/lib/streaming/helpers/__tests__/cap-historical-answer-text.test.ts
new file mode 100644
index 00000000..e8290508
--- /dev/null
+++ b/lib/streaming/helpers/__tests__/cap-historical-answer-text.test.ts
@@ -0,0 +1,378 @@
+import type { UIMessage } from 'ai'
+import { describe, expect, it } from 'vitest'
+
+import {
+ parseAnswerTextExemptTurns,
+ parseAnswerTextLimit
+} from '../cap-historical-answer-text'
+import { compactHistoricalMessages } from '../compact-historical-messages'
+
+const LIMIT = 20
+const PLACEHOLDER_PREFIX = '[Earlier answer text omitted'
+
+function userTurn(id: string, text = 'Question'): UIMessage {
+ return {
+ id,
+ role: 'user',
+ parts: [{ type: 'text', text }]
+ } as UIMessage
+}
+
+function assistantTurn(id: string, text: string): UIMessage {
+ return {
+ id,
+ role: 'assistant',
+ parts: [{ type: 'text', text }]
+ } as UIMessage
+}
+
+function emptyAssistantTurn(id: string): UIMessage {
+ return assistantTurn(id, ' ')
+}
+
+function toolOnlyAssistantTurn(id: string): UIMessage {
+ return {
+ id,
+ role: 'assistant',
+ parts: [
+ {
+ type: 'tool-search',
+ toolCallId: `call_${id}`,
+ state: 'output-available',
+ input: { query: 'example' },
+ output: { query: 'example', images: [], results: [] }
+ }
+ ]
+ } as unknown as UIMessage
+}
+
+function citedAssistantTurn(id: string, text: string): UIMessage {
+ return {
+ id,
+ role: 'assistant',
+ parts: [
+ {
+ type: 'tool-search',
+ toolCallId: 'call_1',
+ state: 'output-available',
+ input: { query: 'example' },
+ output: {
+ query: 'example',
+ images: [],
+ results: [
+ {
+ title: 'Example source',
+ url: 'https://example.com/a-long-source-path',
+ content:
+ 'Evidence that must remain available after answer capping.'
+ }
+ ]
+ }
+ },
+ { type: 'text', text }
+ ]
+ } as unknown as UIMessage
+}
+
+function thread(answerLengths: number[]): UIMessage[] {
+ return answerLengths.flatMap((length, index) => [
+ userTurn(`u${index}`),
+ assistantTurn(`a${index}`, `${index}:${'x'.repeat(length)}`)
+ ])
+}
+
+function compact(
+ messages: UIMessage[],
+ maxChars = LIMIT,
+ exemptCount = 0
+): UIMessage[] {
+ return compactHistoricalMessages(messages, { maxChars, exemptCount })
+}
+
+function textParts(message: UIMessage): string[] {
+ return message.parts.flatMap(part =>
+ part.type === 'text' ? [part.text] : []
+ )
+}
+
+function placeholders(message: UIMessage): string[] {
+ return textParts(message).filter(text => text.startsWith(PLACEHOLDER_PREFIX))
+}
+
+describe('historical answer text cap', () => {
+ it('returns answers under the limit without a placeholder', () => {
+ const messages = thread([5, 5]).concat(userTurn('current'))
+ const capped = compact(messages)
+
+ expect(capped.flatMap(placeholders)).toEqual([])
+ expect(textParts(capped[1])).toEqual(textParts(messages[1]))
+ expect(textParts(capped[3])).toEqual(textParts(messages[3]))
+ })
+
+ it('truncates an older long answer and adds one placeholder', () => {
+ const messages = thread([100, 5]).concat(userTurn('current'))
+ const capped = compact(messages, LIMIT, 1)
+ const oldAnswer = capped[1]
+
+ expect(textParts(oldAnswer)[0]).toHaveLength(LIMIT)
+ expect(placeholders(oldAnswer)).toHaveLength(1)
+ })
+
+ it('exempts the newest historical assistant answers', () => {
+ const messages = thread([100, 100, 100, 100]).concat(userTurn('current'))
+ const capped = compact(messages, LIMIT, 2)
+
+ expect(placeholders(capped[1])).toHaveLength(1)
+ expect(placeholders(capped[3])).toHaveLength(1)
+ expect(placeholders(capped[5])).toEqual([])
+ expect(placeholders(capped[7])).toEqual([])
+ })
+
+ it('does not count an empty assistant between answers as exempt', () => {
+ const messages = [
+ assistantTurn('a0', 'x'.repeat(100)),
+ emptyAssistantTurn('empty'),
+ assistantTurn('a1', 'x'.repeat(100)),
+ assistantTurn('a2', 'x'.repeat(100)),
+ userTurn('current')
+ ]
+ const capped = compact(messages, LIMIT, 2)
+
+ expect(capped.map(message => message.id)).toEqual([
+ 'a0',
+ 'a1',
+ 'a2',
+ 'current'
+ ])
+ expect(placeholders(capped[0])).toHaveLength(1)
+ expect(placeholders(capped[1])).toEqual([])
+ expect(placeholders(capped[2])).toEqual([])
+ })
+
+ it('ignores trailing empty and tool-only assistants for exemptions', () => {
+ const messages = [
+ assistantTurn('a0', 'x'.repeat(100)),
+ assistantTurn('a1', 'x'.repeat(100)),
+ assistantTurn('a2', 'x'.repeat(100)),
+ emptyAssistantTurn('empty-1'),
+ toolOnlyAssistantTurn('tool-only'),
+ emptyAssistantTurn('empty-2'),
+ userTurn('current')
+ ]
+ const capped = compact(messages, LIMIT, 2)
+
+ expect(capped.map(message => message.id)).toEqual([
+ 'a0',
+ 'a1',
+ 'a2',
+ 'current'
+ ])
+ expect(placeholders(capped[0])).toHaveLength(1)
+ expect(placeholders(capped[1])).toEqual([])
+ expect(placeholders(capped[2])).toEqual([])
+ })
+
+ it('leaves messages at or after the history boundary uncapped', () => {
+ const messages = [
+ ...thread([100]),
+ userTurn('current'),
+ assistantTurn('current-assistant', 'x'.repeat(100))
+ ]
+ const capped = compact(messages)
+
+ expect(capped.at(-2)).toBe(messages.at(-2))
+ expect(textParts(capped.at(-1)!)).toEqual(['x'.repeat(100)])
+ expect(placeholders(capped.at(-1)!)).toEqual([])
+ })
+
+ it('keeps capped prefix text stable as the thread grows', () => {
+ const fullThread = thread(Array.from({ length: 8 }, () => 100))
+ const rendered = [4, 6, 8].map(turnCount =>
+ compact(
+ fullThread.slice(0, turnCount * 2).concat(userTurn('current')),
+ LIMIT,
+ 2
+ )
+ )
+
+ for (let assistantIndex = 0; assistantIndex < 2; assistantIndex++) {
+ const messageIndex = assistantIndex * 2 + 1
+ const baseline = JSON.stringify(textParts(rendered[0][messageIndex]))
+
+ for (const messages of rendered.slice(1)) {
+ expect(JSON.stringify(textParts(messages[messageIndex]))).toBe(baseline)
+ }
+ }
+ })
+
+ it('moves the divergence point only within the exempt tail', () => {
+ const fullThread = thread(Array.from({ length: 8 }, () => 100))
+ const render = (turnCount: number) =>
+ compact(
+ fullThread.slice(0, turnCount * 2).concat(userTurn('current')),
+ LIMIT,
+ 2
+ )
+ const shorter = render(4)
+ const longer = render(6)
+
+ expect(placeholders(shorter[5])).toEqual([])
+ expect(placeholders(longer[5])).toHaveLength(1)
+
+ for (const messageIndex of [0, 1, 2, 3, 4]) {
+ expect(textParts(longer[messageIndex])).toEqual(
+ textParts(shorter[messageIndex])
+ )
+ }
+ })
+
+ it('keeps prefix stability with empty assistant records', () => {
+ const replayableTurns = Array.from({ length: 6 }, (_, index) => [
+ assistantTurn(`a${index}`, 'x'.repeat(100)),
+ emptyAssistantTurn(`empty-${index}`)
+ ]).flat()
+ const render = (replayableCount: number) =>
+ compact(
+ replayableTurns
+ .slice(0, replayableCount * 2)
+ .concat(userTurn('current')),
+ LIMIT,
+ 2
+ )
+ const shorter = render(4)
+ const longer = render(6)
+
+ expect(placeholders(shorter[2])).toEqual([])
+ expect(placeholders(longer[2])).toHaveLength(1)
+
+ for (const messageIndex of [0, 1]) {
+ expect(textParts(longer[messageIndex])).toEqual(
+ textParts(shorter[messageIndex])
+ )
+ }
+ })
+
+ it('caps total answer text across multiple text parts', () => {
+ const assistant = {
+ id: 'a0',
+ role: 'assistant',
+ parts: [
+ { type: 'text', text: '1234567890' },
+ { type: 'data-status', data: { status: 'done' } },
+ { type: 'text', text: 'abcdefghijklmno' },
+ { type: 'text', text: 'dropped' }
+ ]
+ } as unknown as UIMessage
+ const originalParts = structuredClone(assistant.parts)
+ const capped = compact([assistant, userTurn('current')])[0]
+
+ expect(textParts(capped)).toEqual([
+ '1234567890',
+ 'abcdefghij',
+ expect.stringContaining(PLACEHOLDER_PREFIX)
+ ])
+ expect(capped.parts).not.toContain(assistant.parts[1])
+ expect(assistant.parts).toEqual(originalParts)
+ })
+
+ it('does not split a surrogate pair at the boundary', () => {
+ const answer = assistantTurn('a0', `${'x'.repeat(19)}😀after`)
+ const capped = compact([answer, userTurn('current')])[0]
+
+ expect(textParts(capped)[0]).toBe('x'.repeat(19))
+ expect(textParts(capped).join('')).not.toMatch(
+ /[\uD800-\uDBFF](?![\uDC00-\uDFFF])|(?<![\uD800-\uDBFF])[\uDC00-\uDFFF]/
+ )
+ })
+
+ it('cuts at whitespace and strips incomplete compact and expanded citations', () => {
+ const whitespaceAnswer = assistantTurn(
+ 'whitespace',
+ 'complete words unfinishedword'
+ )
+ const compactCitationAnswer = assistantTurn(
+ 'compact-citation',
+ 'answer[12](#toolu_01abcdef) trailing'
+ )
+ const expandedCitationAnswer = citedAssistantTurn(
+ 'expanded-citation',
+ 'answer[1](#call_1) trailing'
+ )
+ const capped = compact(
+ [
+ whitespaceAnswer,
+ compactCitationAnswer,
+ expandedCitationAnswer,
+ userTurn('current')
+ ],
+ 24
+ )
+
+ expect(textParts(capped[0])[0]).toBe('complete words')
+ expect(textParts(capped[1])[0]).toBe('answer')
+ expect(textParts(capped[2])[0]).toBe('answer')
+ })
+
+ it('measures the cap after compact citations are expanded', () => {
+ const answer = citedAssistantTurn('cited', 'Answer[1](#call_1)')
+ const compactText = textParts(answer).at(-1)!
+ const maxChars = compactText.length
+ const expandedText = textParts(
+ compact([answer, userTurn('current')], 0)[0]
+ )[0]
+ const capped = compact([answer, userTurn('current')], maxChars)[0]
+ const [answerText] = textParts(capped)
+
+ expect(compactText).toHaveLength(maxChars)
+ expect(expandedText).toBe(
+ 'Answer[example](https://example.com/a-long-source-path)'
+ )
+ expect(expandedText.length).toBeGreaterThan(maxChars)
+ expect(answerText).toBe('Answer')
+ expect(answerText.length).toBeLessThanOrEqual(maxChars)
+ expect(placeholders(capped)).toHaveLength(1)
+ })
+
+ it('adds unchanged source context after the capped answer budget', () => {
+ const answer = citedAssistantTurn(
+ 'cited',
+ `Long answer text ${'x'.repeat(50)} [1](#call_1)`
+ )
+ const messages = [answer, userTurn('current')]
+ const capped = compact(messages, 10)[0]
+ const uncapped = compact(messages, 0)[0]
+ const cappedTexts = textParts(capped)
+ const sourceContext = cappedTexts.at(-1)!
+
+ expect(cappedTexts[0].length).toBeLessThanOrEqual(10)
+ expect(placeholders(capped)).toHaveLength(1)
+ expect(sourceContext).toBe(textParts(uncapped).at(-1))
+ expect(sourceContext.length).toBeGreaterThan(10)
+ expect(sourceContext).toContain('Evidence that must remain available')
+ expect(sourceContext).toContain('https://example.com/a-long-source-path')
+ })
+
+ it('disables the cap with an explicit zero limit', () => {
+ const messages = thread([100]).concat(userTurn('current'))
+ const capped = compact(messages, 0)
+
+ expect(textParts(capped[1])).toEqual(textParts(messages[1]))
+ expect(placeholders(capped[1])).toEqual([])
+ })
+})
+
+describe('answer text configuration parsing', () => {
+ it('falls back to defaults for empty and malformed values', () => {
+ for (const raw of [undefined, '', ' ', 'nope', '-1', 'Infinity']) {
+ expect(parseAnswerTextLimit(raw)).toBe(12_000)
+ expect(parseAnswerTextExemptTurns(raw)).toBe(2)
+ }
+ })
+
+ it('accepts explicit zero and floors positive values', () => {
+ expect(parseAnswerTextLimit('0')).toBe(0)
+ expect(parseAnswerTextLimit('12.9')).toBe(12)
+ expect(parseAnswerTextExemptTurns('0')).toBe(0)
+ expect(parseAnswerTextExemptTurns('3.9')).toBe(3)
+ })
+})
diff --git a/lib/streaming/helpers/cap-historical-answer-text.ts b/lib/streaming/helpers/cap-historical-answer-text.ts
new file mode 100644
index 00000000..15442f12
--- /dev/null
+++ b/lib/streaming/helpers/cap-historical-answer-text.ts
@@ -0,0 +1,124 @@
+import type { UIMessage } from 'ai'
+
+import { sliceWithoutSplittingSurrogatePair } from './slice-without-splitting-surrogate-pair'
+
+const DEFAULT_ANSWER_TEXT_LIMIT = 12_000
+const DEFAULT_ANSWER_TEXT_EXEMPT_TURNS = 2
+const INCOMPLETE_CITATION_PATTERN =
+ /(?:\[\s*\d+(?:\s*\]?(?:\s*\(\s*(?:#[^)]*)?)?)?|\[[^\]\r\n]+\]\(\s*[^)\r\n]*)$/
+const OMITTED_ANSWER_PLACEHOLDER =
+ '[Earlier answer text omitted from replay. The full answer remains visible in the conversation.]'
+
+type AnswerTextPart = { type: 'text'; text: string }
+
+export function hasReplayableAnswerText(message: UIMessage): boolean {
+ return (
+ message.role === 'assistant' &&
+ message.parts.some(part => part.type === 'text' && part.text.trim())
+ )
+}
+
+export function parseAnswerTextLimit(raw: string | undefined): number {
+ const value = raw?.trim()
+ if (!value) return DEFAULT_ANSWER_TEXT_LIMIT
+
+ const parsed = Number(value)
+ if (!Number.isFinite(parsed) || parsed < 0) {
+ return DEFAULT_ANSWER_TEXT_LIMIT
+ }
+
+ return parsed === 0 ? 0 : Math.max(1, Math.floor(parsed))
+}
+
+export const HISTORY_ANSWER_TEXT_LIMIT = parseAnswerTextLimit(
+ process.env.HISTORY_ANSWER_TEXT_LIMIT
+)
+
+export function parseAnswerTextExemptTurns(raw: string | undefined): number {
+ const value = raw?.trim()
+ if (!value) return DEFAULT_ANSWER_TEXT_EXEMPT_TURNS
+
+ const parsed = Number(value)
+ if (!Number.isFinite(parsed) || parsed < 0) {
+ return DEFAULT_ANSWER_TEXT_EXEMPT_TURNS
+ }
+
+ return parsed === 0 ? 0 : Math.max(1, Math.floor(parsed))
+}
+
+export const HISTORY_ANSWER_TEXT_EXEMPT_TURNS = parseAnswerTextExemptTurns(
+ process.env.HISTORY_ANSWER_TEXT_EXEMPT_TURNS
+)
+
+function truncateAnswerText(value: string, maxChars: number): string {
+ let truncated = sliceWithoutSplittingSurrogatePair(value, maxChars)
+ const whitespaceIndex = truncated.search(/\s+\S*$/)
+
+ if (whitespaceIndex > 0) {
+ truncated = truncated.slice(0, whitespaceIndex)
+ }
+
+ return truncated.replace(INCOMPLETE_CITATION_PATTERN, '').trimEnd()
+}
+
+export function selectHistoricalAssistantMessagesToCap(
+ messages: UIMessage[],
+ exemptCount: number
+): Set<UIMessage> {
+ const currentTurnIndex = messages.findLastIndex(
+ message => message.role === 'user'
+ )
+ const historyEnd =
+ currentTurnIndex === -1 ? messages.length : currentTurnIndex
+ const assistantCount = messages
+ .slice(0, historyEnd)
+ .filter(hasReplayableAnswerText).length
+ const cappedAssistantCount = Math.max(
+ 0,
+ assistantCount - Math.max(0, Math.floor(exemptCount))
+ )
+
+ let seenAssistants = 0
+ const selected = new Set<UIMessage>()
+ for (const [index, message] of messages.entries()) {
+ if (index >= historyEnd) break
+ if (!hasReplayableAnswerText(message)) continue
+
+ seenAssistants += 1
+ if (seenAssistants > cappedAssistantCount) break
+ selected.add(message)
+ }
+
+ return selected
+}
+
+export function capAnswerTextParts(
+ textParts: AnswerTextPart[],
+ maxChars: number
+): AnswerTextPart[] {
+ if (
+ maxChars <= 0 ||
+ textParts.reduce((total, part) => total + part.text.length, 0) <= maxChars
+ ) {
+ return textParts
+ }
+
+ let remaining = maxChars
+ const cappedParts: AnswerTextPart[] = []
+ for (const part of textParts) {
+ if (remaining <= 0) continue
+
+ if (part.text.length <= remaining) {
+ remaining -= part.text.length
+ cappedParts.push(part)
+ continue
+ }
+
+ const text = truncateAnswerText(part.text, remaining)
+ remaining = 0
+ if (text) cappedParts.push({ ...part, text })
+ }
+
+ cappedParts.push({ type: 'text', text: OMITTED_ANSWER_PLACEHOLDER })
+ return cappedParts
+}
diff --git a/lib/streaming/helpers/compact-historical-messages.ts b/lib/streaming/helpers/compact-historical-messages.ts
index cfddb1c1..c7b3aea7 100644
--- a/lib/streaming/helpers/compact-historical-messages.ts
+++ b/lib/streaming/helpers/compact-historical-messages.ts
@@ -3,6 +3,15 @@ import type { UIMessage } from 'ai'
import type { SearchResultItem } from '@/lib/types'
import { extractCitationMaps, processCitations } from '@/lib/utils/citation'
+import {
+ capAnswerTextParts,
+ hasReplayableAnswerText,
+ HISTORY_ANSWER_TEXT_EXEMPT_TURNS,
+ HISTORY_ANSWER_TEXT_LIMIT,
+ selectHistoricalAssistantMessagesToCap
+} from './cap-historical-answer-text'
+import { sliceWithoutSplittingSurrogatePair } from './slice-without-splitting-surrogate-pair'
+
// Applies per assistant turn. Source context is attached to every cited turn
// and never removed later, so this bound is what keeps a long conversation's
// accumulated evidence from crowding out the context window.
@@ -25,20 +34,6 @@ function normalizeInlineText(value: string): string {
.trim()
}
-function sliceWithoutSplittingSurrogatePair(
- value: string,
- maxChars: number
-): string {
- if (value.length <= maxChars) return value
-
- const sliced = value.slice(0, maxChars)
- const lastCodeUnit = sliced.charCodeAt(sliced.length - 1)
-
- return lastCodeUnit >= 0xd800 && lastCodeUnit <= 0xdbff
- ? sliced.slice(0, -1)
- : sliced
-}
-
function truncateInlineText(value: string, maxChars: number): string {
if (value.length <= maxChars) return value
if (maxChars <= 1) return sliceWithoutSplittingSurrogatePair(value, maxChars)
@@ -214,19 +209,34 @@ Entries correspond to cited sources in order. Their URLs remain in the preceding
* context. The current request's ToolLoopAgent messages do not pass through
* this function, so its active reasoning/tool sequence remains intact.
*
- * A message's output depends only on that message. Selecting by position in
- * the conversation instead would rewrite already-sent history as turns are
- * appended, which both changes what the model saw earlier and invalidates the
- * provider's prompt cache from that point on.
+ * A message's output depends only on that message and, within the newest
+ * exempt assistant messages, its bounded distance from the end. Appending
+ * turns can move the divergence point only within that tail, so the stable
+ * prompt prefix remains independent of total thread length.
*/
-export function compactHistoricalMessages(messages: UIMessage[]): UIMessage[] {
+export function compactHistoricalMessages(
+ messages: UIMessage[],
+ answerTextCap: { maxChars?: number; exemptCount?: number } = {}
+): UIMessage[] {
+ const maxChars = answerTextCap.maxChars ?? HISTORY_ANSWER_TEXT_LIMIT
+ const exemptCount =
+ answerTextCap.exemptCount ?? HISTORY_ANSWER_TEXT_EXEMPT_TURNS
+ const messagesToCap =
+ maxChars > 0
+ ? selectHistoricalAssistantMessagesToCap(messages, exemptCount)
+ : new Set<UIMessage>()
+
return messages.flatMap(message => {
if (message.role !== 'assistant') {
return [message]
}
+ if (!hasReplayableAnswerText(message)) {
+ return []
+ }
+
const citationMaps = extractCitationMaps(message)
- const textParts = message.parts.flatMap(part => {
+ let textParts = message.parts.flatMap(part => {
if (part.type !== 'text' || !part.text.trim()) {
return []
}
@@ -241,8 +251,8 @@ export function compactHistoricalMessages(messages: UIMessage[]): UIMessage[] {
]
})
- if (textParts.length === 0) {
- return []
+ if (messagesToCap.has(message)) {
+ textParts = capAnswerTextParts(textParts, maxChars)
}
const sourceContext = createSourceContext(
diff --git a/lib/streaming/helpers/slice-without-splitting-surrogate-pair.ts b/lib/streaming/helpers/slice-without-splitting-surrogate-pair.ts
new file mode 100644
index 00000000..6b5fa192
--- /dev/null
+++ b/lib/streaming/helpers/slice-without-splitting-surrogate-pair.ts
@@ -0,0 +1,13 @@
+export function sliceWithoutSplittingSurrogatePair(
+ value: string,
+ maxChars: number
+): string {
+ if (value.length <= maxChars) return value
+
+ const sliced = value.slice(0, maxChars)
+ const lastCodeUnit = sliced.charCodeAt(sliced.length - 1)
+
+ return lastCodeUnit >= 0xd800 && lastCodeUnit <= 0xdbff
+ ? sliced.slice(0, -1)
+ : sliced
+}