An AI-powered search engine with a generative UI
TypeScript generative-aigenerative-uinextjsollamareactsearxngshadcn-uitailwindcsstavilytypescriptupstashvercel-ai-sdk
fix(streaming): cap replayed historical answer text (#996)
Yoshiki Miura244336f2cf15
diff --git a/lib/streaming/helpers/__tests__/cap-historical-answer-text.test.ts b/lib/streaming/helpers/__tests__/cap-historical-answer-text.test.tsnew file mode 100644index 00000000..e8290508--- /dev/null+++ b/lib/streaming/helpers/__tests__/cap-historical-answer-text.test.ts@@ -0,0 +1,378 @@+import type { UIMessage } from 'ai'+import { describe, expect, it } from 'vitest'++import {+ parseAnswerTextExemptTurns,+ parseAnswerTextLimit+} from '../cap-historical-answer-text'+import { compactHistoricalMessages } from '../compact-historical-messages'++const LIMIT = 20+const PLACEHOLDER_PREFIX = '[Earlier answer text omitted'++function userTurn(id: string, text = 'Question'): UIMessage {+ return {+ id,+ role: 'user',+ parts: [{ type: 'text', text }]+ } as UIMessage+}++function assistantTurn(id: string, text: string): UIMessage {+ return {+ id,+ role: 'assistant',+ parts: [{ type: 'text', text }]+ } as UIMessage+}++function emptyAssistantTurn(id: string): UIMessage {+ return assistantTurn(id, ' ')+}++function toolOnlyAssistantTurn(id: string): UIMessage {+ return {+ id,+ role: 'assistant',+ parts: [+ {+ type: 'tool-search',+ toolCallId: `call_${id}`,+ state: 'output-available',+ input: { query: 'example' },+ output: { query: 'example', images: [], results: [] }+ }+ ]+ } as unknown as UIMessage+}++function citedAssistantTurn(id: string, text: string): UIMessage {+ return {+ id,+ role: 'assistant',+ parts: [+ {+ type: 'tool-search',+ toolCallId: 'call_1',+ state: 'output-available',+ input: { query: 'example' },+ output: {+ query: 'example',+ images: [],+ results: [+ {+ title: 'Example source',+ url: 'https://example.com/a-long-source-path',+ content:+ 'Evidence that must remain available after answer capping.'+ }+ ]+ }+ },+ { type: 'text', text }+ ]+ } as unknown as UIMessage+}++function thread(answerLengths: number[]): UIMessage[] {+ return answerLengths.flatMap((length, index) => [+ userTurn(`u${index}`),+ assistantTurn(`a${index}`, `${index}:${'x'.repeat(length)}`)+ ])+}++function compact(+ messages: UIMessage[],+ maxChars = LIMIT,+ exemptCount = 0+): UIMessage[] {+ return compactHistoricalMessages(messages, { maxChars, exemptCount })+}++function textParts(message: UIMessage): string[] {+ return message.parts.flatMap(part =>+ part.type === 'text' ? [part.text] : []+ )+}++function placeholders(message: UIMessage): string[] {+ return textParts(message).filter(text => text.startsWith(PLACEHOLDER_PREFIX))+}++describe('historical answer text cap', () => {+ it('returns answers under the limit without a placeholder', () => {+ const messages = thread([5, 5]).concat(userTurn('current'))+ const capped = compact(messages)++ expect(capped.flatMap(placeholders)).toEqual([])+ expect(textParts(capped[1])).toEqual(textParts(messages[1]))+ expect(textParts(capped[3])).toEqual(textParts(messages[3]))+ })++ it('truncates an older long answer and adds one placeholder', () => {+ const messages = thread([100, 5]).concat(userTurn('current'))+ const capped = compact(messages, LIMIT, 1)+ const oldAnswer = capped[1]++ expect(textParts(oldAnswer)[0]).toHaveLength(LIMIT)+ expect(placeholders(oldAnswer)).toHaveLength(1)+ })++ it('exempts the newest historical assistant answers', () => {+ const messages = thread([100, 100, 100, 100]).concat(userTurn('current'))+ const capped = compact(messages, LIMIT, 2)++ expect(placeholders(capped[1])).toHaveLength(1)+ expect(placeholders(capped[3])).toHaveLength(1)+ expect(placeholders(capped[5])).toEqual([])+ expect(placeholders(capped[7])).toEqual([])+ })++ it('does not count an empty assistant between answers as exempt', () => {+ const messages = [+ assistantTurn('a0', 'x'.repeat(100)),+ emptyAssistantTurn('empty'),+ assistantTurn('a1', 'x'.repeat(100)),+ assistantTurn('a2', 'x'.repeat(100)),+ userTurn('current')+ ]+ const capped = compact(messages, LIMIT, 2)++ expect(capped.map(message => message.id)).toEqual([+ 'a0',+ 'a1',+ 'a2',+ 'current'+ ])+ expect(placeholders(capped[0])).toHaveLength(1)+ expect(placeholders(capped[1])).toEqual([])+ expect(placeholders(capped[2])).toEqual([])+ })++ it('ignores trailing empty and tool-only assistants for exemptions', () => {+ const messages = [+ assistantTurn('a0', 'x'.repeat(100)),+ assistantTurn('a1', 'x'.repeat(100)),+ assistantTurn('a2', 'x'.repeat(100)),+ emptyAssistantTurn('empty-1'),+ toolOnlyAssistantTurn('tool-only'),+ emptyAssistantTurn('empty-2'),+ userTurn('current')+ ]+ const capped = compact(messages, LIMIT, 2)++ expect(capped.map(message => message.id)).toEqual([+ 'a0',+ 'a1',+ 'a2',+ 'current'+ ])+ expect(placeholders(capped[0])).toHaveLength(1)+ expect(placeholders(capped[1])).toEqual([])+ expect(placeholders(capped[2])).toEqual([])+ })++ it('leaves messages at or after the history boundary uncapped', () => {+ const messages = [+ ...thread([100]),+ userTurn('current'),+ assistantTurn('current-assistant', 'x'.repeat(100))+ ]+ const capped = compact(messages)++ expect(capped.at(-2)).toBe(messages.at(-2))+ expect(textParts(capped.at(-1)!)).toEqual(['x'.repeat(100)])+ expect(placeholders(capped.at(-1)!)).toEqual([])+ })++ it('keeps capped prefix text stable as the thread grows', () => {+ const fullThread = thread(Array.from({ length: 8 }, () => 100))+ const rendered = [4, 6, 8].map(turnCount =>+ compact(+ fullThread.slice(0, turnCount * 2).concat(userTurn('current')),+ LIMIT,+ 2+ )+ )++ for (let assistantIndex = 0; assistantIndex < 2; assistantIndex++) {+ const messageIndex = assistantIndex * 2 + 1+ const baseline = JSON.stringify(textParts(rendered[0][messageIndex]))++ for (const messages of rendered.slice(1)) {+ expect(JSON.stringify(textParts(messages[messageIndex]))).toBe(baseline)+ }+ }+ })++ it('moves the divergence point only within the exempt tail', () => {+ const fullThread = thread(Array.from({ length: 8 }, () => 100))+ const render = (turnCount: number) =>+ compact(+ fullThread.slice(0, turnCount * 2).concat(userTurn('current')),+ LIMIT,+ 2+ )+ const shorter = render(4)+ const longer = render(6)++ expect(placeholders(shorter[5])).toEqual([])+ expect(placeholders(longer[5])).toHaveLength(1)++ for (const messageIndex of [0, 1, 2, 3, 4]) {+ expect(textParts(longer[messageIndex])).toEqual(+ textParts(shorter[messageIndex])+ )+ }+ })++ it('keeps prefix stability with empty assistant records', () => {+ const replayableTurns = Array.from({ length: 6 }, (_, index) => [+ assistantTurn(`a${index}`, 'x'.repeat(100)),+ emptyAssistantTurn(`empty-${index}`)+ ]).flat()+ const render = (replayableCount: number) =>+ compact(+ replayableTurns+ .slice(0, replayableCount * 2)+ .concat(userTurn('current')),+ LIMIT,+ 2+ )+ const shorter = render(4)+ const longer = render(6)++ expect(placeholders(shorter[2])).toEqual([])+ expect(placeholders(longer[2])).toHaveLength(1)++ for (const messageIndex of [0, 1]) {+ expect(textParts(longer[messageIndex])).toEqual(+ textParts(shorter[messageIndex])+ )+ }+ })++ it('caps total answer text across multiple text parts', () => {+ const assistant = {+ id: 'a0',+ role: 'assistant',+ parts: [+ { type: 'text', text: '1234567890' },+ { type: 'data-status', data: { status: 'done' } },+ { type: 'text', text: 'abcdefghijklmno' },+ { type: 'text', text: 'dropped' }+ ]+ } as unknown as UIMessage+ const originalParts = structuredClone(assistant.parts)+ const capped = compact([assistant, userTurn('current')])[0]++ expect(textParts(capped)).toEqual([+ '1234567890',+ 'abcdefghij',+ expect.stringContaining(PLACEHOLDER_PREFIX)+ ])+ expect(capped.parts).not.toContain(assistant.parts[1])+ expect(assistant.parts).toEqual(originalParts)+ })++ it('does not split a surrogate pair at the boundary', () => {+ const answer = assistantTurn('a0', `${'x'.repeat(19)}😀after`)+ const capped = compact([answer, userTurn('current')])[0]++ expect(textParts(capped)[0]).toBe('x'.repeat(19))+ expect(textParts(capped).join('')).not.toMatch(+ /[\uD800-\uDBFF](?![\uDC00-\uDFFF])|(?<![\uD800-\uDBFF])[\uDC00-\uDFFF]/+ )+ })++ it('cuts at whitespace and strips incomplete compact and expanded citations', () => {+ const whitespaceAnswer = assistantTurn(+ 'whitespace',+ 'complete words unfinishedword'+ )+ const compactCitationAnswer = assistantTurn(+ 'compact-citation',+ 'answer[12](#toolu_01abcdef) trailing'+ )+ const expandedCitationAnswer = citedAssistantTurn(+ 'expanded-citation',+ 'answer[1](#call_1) trailing'+ )+ const capped = compact(+ [+ whitespaceAnswer,+ compactCitationAnswer,+ expandedCitationAnswer,+ userTurn('current')+ ],+ 24+ )++ expect(textParts(capped[0])[0]).toBe('complete words')+ expect(textParts(capped[1])[0]).toBe('answer')+ expect(textParts(capped[2])[0]).toBe('answer')+ })++ it('measures the cap after compact citations are expanded', () => {+ const answer = citedAssistantTurn('cited', 'Answer[1](#call_1)')+ const compactText = textParts(answer).at(-1)!+ const maxChars = compactText.length+ const expandedText = textParts(+ compact([answer, userTurn('current')], 0)[0]+ )[0]+ const capped = compact([answer, userTurn('current')], maxChars)[0]+ const [answerText] = textParts(capped)++ expect(compactText).toHaveLength(maxChars)+ expect(expandedText).toBe(+ 'Answer[example](https://example.com/a-long-source-path)'+ )+ expect(expandedText.length).toBeGreaterThan(maxChars)+ expect(answerText).toBe('Answer')+ expect(answerText.length).toBeLessThanOrEqual(maxChars)+ expect(placeholders(capped)).toHaveLength(1)+ })++ it('adds unchanged source context after the capped answer budget', () => {+ const answer = citedAssistantTurn(+ 'cited',+ `Long answer text ${'x'.repeat(50)} [1](#call_1)`+ )+ const messages = [answer, userTurn('current')]+ const capped = compact(messages, 10)[0]+ const uncapped = compact(messages, 0)[0]+ const cappedTexts = textParts(capped)+ const sourceContext = cappedTexts.at(-1)!++ expect(cappedTexts[0].length).toBeLessThanOrEqual(10)+ expect(placeholders(capped)).toHaveLength(1)+ expect(sourceContext).toBe(textParts(uncapped).at(-1))+ expect(sourceContext.length).toBeGreaterThan(10)+ expect(sourceContext).toContain('Evidence that must remain available')+ expect(sourceContext).toContain('https://example.com/a-long-source-path')+ })++ it('disables the cap with an explicit zero limit', () => {+ const messages = thread([100]).concat(userTurn('current'))+ const capped = compact(messages, 0)++ expect(textParts(capped[1])).toEqual(textParts(messages[1]))+ expect(placeholders(capped[1])).toEqual([])+ })+})++describe('answer text configuration parsing', () => {+ it('falls back to defaults for empty and malformed values', () => {+ for (const raw of [undefined, '', ' ', 'nope', '-1', 'Infinity']) {+ expect(parseAnswerTextLimit(raw)).toBe(12_000)+ expect(parseAnswerTextExemptTurns(raw)).toBe(2)+ }+ })++ it('accepts explicit zero and floors positive values', () => {+ expect(parseAnswerTextLimit('0')).toBe(0)+ expect(parseAnswerTextLimit('12.9')).toBe(12)+ expect(parseAnswerTextExemptTurns('0')).toBe(0)+ expect(parseAnswerTextExemptTurns('3.9')).toBe(3)+ })+})diff --git a/lib/streaming/helpers/cap-historical-answer-text.ts b/lib/streaming/helpers/cap-historical-answer-text.tsnew file mode 100644index 00000000..15442f12--- /dev/null+++ b/lib/streaming/helpers/cap-historical-answer-text.ts@@ -0,0 +1,124 @@+import type { UIMessage } from 'ai'++import { sliceWithoutSplittingSurrogatePair } from './slice-without-splitting-surrogate-pair'++const DEFAULT_ANSWER_TEXT_LIMIT = 12_000+const DEFAULT_ANSWER_TEXT_EXEMPT_TURNS = 2+const INCOMPLETE_CITATION_PATTERN =+ /(?:\[\s*\d+(?:\s*\]?(?:\s*\(\s*(?:#[^)]*)?)?)?|\[[^\]\r\n]+\]\(\s*[^)\r\n]*)$/+const OMITTED_ANSWER_PLACEHOLDER =+ '[Earlier answer text omitted from replay. The full answer remains visible in the conversation.]'++type AnswerTextPart = { type: 'text'; text: string }++export function hasReplayableAnswerText(message: UIMessage): boolean {+ return (+ message.role === 'assistant' &&+ message.parts.some(part => part.type === 'text' && part.text.trim())+ )+}++export function parseAnswerTextLimit(raw: string | undefined): number {+ const value = raw?.trim()+ if (!value) return DEFAULT_ANSWER_TEXT_LIMIT++ const parsed = Number(value)+ if (!Number.isFinite(parsed) || parsed < 0) {+ return DEFAULT_ANSWER_TEXT_LIMIT+ }++ return parsed === 0 ? 0 : Math.max(1, Math.floor(parsed))+}++export const HISTORY_ANSWER_TEXT_LIMIT = parseAnswerTextLimit(+ process.env.HISTORY_ANSWER_TEXT_LIMIT+)++export function parseAnswerTextExemptTurns(raw: string | undefined): number {+ const value = raw?.trim()+ if (!value) return DEFAULT_ANSWER_TEXT_EXEMPT_TURNS++ const parsed = Number(value)+ if (!Number.isFinite(parsed) || parsed < 0) {+ return DEFAULT_ANSWER_TEXT_EXEMPT_TURNS+ }++ return parsed === 0 ? 0 : Math.max(1, Math.floor(parsed))+}++export const HISTORY_ANSWER_TEXT_EXEMPT_TURNS = parseAnswerTextExemptTurns(+ process.env.HISTORY_ANSWER_TEXT_EXEMPT_TURNS+)++function truncateAnswerText(value: string, maxChars: number): string {+ let truncated = sliceWithoutSplittingSurrogatePair(value, maxChars)+ const whitespaceIndex = truncated.search(/\s+\S*$/)++ if (whitespaceIndex > 0) {+ truncated = truncated.slice(0, whitespaceIndex)+ }++ return truncated.replace(INCOMPLETE_CITATION_PATTERN, '').trimEnd()+}++export function selectHistoricalAssistantMessagesToCap(+ messages: UIMessage[],+ exemptCount: number+): Set<UIMessage> {+ const currentTurnIndex = messages.findLastIndex(+ message => message.role === 'user'+ )+ const historyEnd =+ currentTurnIndex === -1 ? messages.length : currentTurnIndex+ const assistantCount = messages+ .slice(0, historyEnd)+ .filter(hasReplayableAnswerText).length+ const cappedAssistantCount = Math.max(+ 0,+ assistantCount - Math.max(0, Math.floor(exemptCount))+ )++ let seenAssistants = 0+ const selected = new Set<UIMessage>()+ for (const [index, message] of messages.entries()) {+ if (index >= historyEnd) break+ if (!hasReplayableAnswerText(message)) continue++ seenAssistants += 1+ if (seenAssistants > cappedAssistantCount) break+ selected.add(message)+ }++ return selected+}++export function capAnswerTextParts(+ textParts: AnswerTextPart[],+ maxChars: number+): AnswerTextPart[] {+ if (+ maxChars <= 0 ||+ textParts.reduce((total, part) => total + part.text.length, 0) <= maxChars+ ) {+ return textParts+ }++ let remaining = maxChars+ const cappedParts: AnswerTextPart[] = []+ for (const part of textParts) {+ if (remaining <= 0) continue++ if (part.text.length <= remaining) {+ remaining -= part.text.length+ cappedParts.push(part)+ continue+ }++ const text = truncateAnswerText(part.text, remaining)+ remaining = 0+ if (text) cappedParts.push({ ...part, text })+ }++ cappedParts.push({ type: 'text', text: OMITTED_ANSWER_PLACEHOLDER })+ return cappedParts+}diff --git a/lib/streaming/helpers/compact-historical-messages.ts b/lib/streaming/helpers/compact-historical-messages.tsindex cfddb1c1..c7b3aea7 100644--- a/lib/streaming/helpers/compact-historical-messages.ts+++ b/lib/streaming/helpers/compact-historical-messages.ts@@ -3,6 +3,15 @@ import type { UIMessage } from 'ai'import type { SearchResultItem } from '@/lib/types'import { extractCitationMaps, processCitations } from '@/lib/utils/citation'+import {+ capAnswerTextParts,+ hasReplayableAnswerText,+ HISTORY_ANSWER_TEXT_EXEMPT_TURNS,+ HISTORY_ANSWER_TEXT_LIMIT,+ selectHistoricalAssistantMessagesToCap+} from './cap-historical-answer-text'+import { sliceWithoutSplittingSurrogatePair } from './slice-without-splitting-surrogate-pair'+// Applies per assistant turn. Source context is attached to every cited turn// and never removed later, so this bound is what keeps a long conversation's// accumulated evidence from crowding out the context window.@@ -25,20 +34,6 @@ function normalizeInlineText(value: string): string {.trim()}-function sliceWithoutSplittingSurrogatePair(- value: string,- maxChars: number-): string {- if (value.length <= maxChars) return value-- const sliced = value.slice(0, maxChars)- const lastCodeUnit = sliced.charCodeAt(sliced.length - 1)-- return lastCodeUnit >= 0xd800 && lastCodeUnit <= 0xdbff- ? sliced.slice(0, -1)- : sliced-}-function truncateInlineText(value: string, maxChars: number): string {if (value.length <= maxChars) return valueif (maxChars <= 1) return sliceWithoutSplittingSurrogatePair(value, maxChars)@@ -214,19 +209,34 @@ Entries correspond to cited sources in order. Their URLs remain in the preceding* context. The current request's ToolLoopAgent messages do not pass through* this function, so its active reasoning/tool sequence remains intact.*- * A message's output depends only on that message. Selecting by position in- * the conversation instead would rewrite already-sent history as turns are- * appended, which both changes what the model saw earlier and invalidates the- * provider's prompt cache from that point on.+ * A message's output depends only on that message and, within the newest+ * exempt assistant messages, its bounded distance from the end. Appending+ * turns can move the divergence point only within that tail, so the stable+ * prompt prefix remains independent of total thread length.*/-export function compactHistoricalMessages(messages: UIMessage[]): UIMessage[] {+export function compactHistoricalMessages(+ messages: UIMessage[],+ answerTextCap: { maxChars?: number; exemptCount?: number } = {}+): UIMessage[] {+ const maxChars = answerTextCap.maxChars ?? HISTORY_ANSWER_TEXT_LIMIT+ const exemptCount =+ answerTextCap.exemptCount ?? HISTORY_ANSWER_TEXT_EXEMPT_TURNS+ const messagesToCap =+ maxChars > 0+ ? selectHistoricalAssistantMessagesToCap(messages, exemptCount)+ : new Set<UIMessage>()+return messages.flatMap(message => {if (message.role !== 'assistant') {return [message]}+ if (!hasReplayableAnswerText(message)) {+ return []+ }+const citationMaps = extractCitationMaps(message)- const textParts = message.parts.flatMap(part => {+ let textParts = message.parts.flatMap(part => {if (part.type !== 'text' || !part.text.trim()) {return []}@@ -241,8 +251,8 @@ export function compactHistoricalMessages(messages: UIMessage[]): UIMessage[] {]})- if (textParts.length === 0) {- return []+ if (messagesToCap.has(message)) {+ textParts = capAnswerTextParts(textParts, maxChars)}const sourceContext = createSourceContext(diff --git a/lib/streaming/helpers/slice-without-splitting-surrogate-pair.ts b/lib/streaming/helpers/slice-without-splitting-surrogate-pair.tsnew file mode 100644index 00000000..6b5fa192--- /dev/null+++ b/lib/streaming/helpers/slice-without-splitting-surrogate-pair.ts@@ -0,0 +1,13 @@+export function sliceWithoutSplittingSurrogatePair(+ value: string,+ maxChars: number+): string {+ if (value.length <= maxChars) return value++ const sliced = value.slice(0, maxChars)+ const lastCodeUnit = sliced.charCodeAt(sliced.length - 1)++ return lastCodeUnit >= 0xd800 && lastCodeUnit <= 0xdbff+ ? sliced.slice(0, -1)+ : sliced+}