Skip to content

An AI-powered search engine with a generative UI

fix(chat): keep the cold-start trim cut fixed on warm follow-up turns (#1024)

Yoshiki Miurab9df031d6dc1
diff --git a/lib/streaming/helpers/__tests__/trim-cold-start-history.test.ts b/lib/streaming/helpers/__tests__/trim-cold-start-history.test.ts
index cf0cbd6d..b21602a6 100644
--- a/lib/streaming/helpers/__tests__/trim-cold-start-history.test.ts
+++ b/lib/streaming/helpers/__tests__/trim-cold-start-history.test.ts
@@ -87,6 +87,37 @@ describe('trimColdStartHistory', () => {
expect(second.slice(0, first.length)).toEqual(first)
})
+ it('preserves the cold-request prefix after its answer and a warm request', () => {
+ const coldAt = timestamp(4 * WARM_GAP_MS + COLD_GAP_MS)
+ const warmAt = timestamp(5 * WARM_GAP_MS + COLD_GAP_MS)
+ const atColdRequest = [
+ ...Array.from({ length: 5 }, (_, i) =>
+ turn(i, timestamp(i * WARM_GAP_MS))
+ ).flat(),
+ message('u5', 'user', coldAt)
+ ]
+ const options = { limit: 100 }
+ const coldResult = trimColdStartHistory(atColdRequest, {
+ ...options,
+ now: coldAt
+ })
+ const appended = [
+ message('a5', 'assistant', coldAt, [
+ { type: 'text', text: 'large answer '.repeat(1_000) }
+ ]),
+ message('u6', 'user', warmAt)
+ ]
+ const warmResult = trimColdStartHistory(atColdRequest.concat(appended), {
+ ...options,
+ now: warmAt
+ })
+
+ expect(coldResult.trimmedAtCurrentTurn).toBe(true)
+ expect(userIds(coldResult.messages)).toEqual(['u0', 'u4', 'u5'])
+ expect(warmResult.messages).toEqual(coldResult.messages.concat(appended))
+ expect(warmResult.trimmedAtCurrentTurn).toBe(false)
+ })
+
it('leaves fewer than five turns untouched', () => {
const messages = Array.from({ length: 4 }, (_, i) =>
turn(i, timestamp(i * COLD_GAP_MS))
diff --git a/lib/streaming/helpers/trim-cold-start-history.ts b/lib/streaming/helpers/trim-cold-start-history.ts
index 3e733653..6a7785fc 100644
--- a/lib/streaming/helpers/trim-cold-start-history.ts
+++ b/lib/streaming/helpers/trim-cold-start-history.ts
@@ -37,6 +37,7 @@ type Turn = {
messages: UIMessage[]
timestamp?: number
replayed: ReplayedText[]
+ leadingUserReplayed: ReplayedText[]
}
const textEncoder = new TextEncoder()
@@ -91,8 +92,11 @@ function tokenUpperBound(turn: Turn): number {
)
}
-function countTurnTokens(turn: Turn, modelId?: string): number {
- return turn.replayed.reduce(
+function countReplayedTokens(
+ replayed: ReplayedText[],
+ modelId?: string
+): number {
+ return replayed.reduce(
(total, { text, fixedTokens }) =>
total + countTextTokens(text, modelId) + fixedTokens,
0
@@ -130,7 +134,8 @@ function buildTurns(messages: UIMessage[], now: Date): Turn[] {
return {
messages: turnMessages,
timestamp,
- replayed: turnMessages.flatMap(describeMessage)
+ replayed: turnMessages.flatMap(describeMessage),
+ leadingUserReplayed: describeMessage(messages[userIndex])
}
})
}
@@ -153,7 +158,12 @@ export function trimColdStartHistory(
)
if (upperBound <= limit) return { messages, trimmedAtCurrentTurn: false }
- const tokens = turns.map(turn => countTurnTokens(turn, options.modelId))
+ const tokens = turns.map(turn =>
+ countReplayedTokens(turn.replayed, options.modelId)
+ )
+ const leadingUserTokens = turns.map(turn =>
+ countReplayedTokens(turn.leadingUserReplayed, options.modelId)
+ )
let cut = 1
let trimmedAtCurrentTurn = false
@@ -167,8 +177,8 @@ export function trimColdStartHistory(
if (!cold || i + 1 < MIN_TURNS) continue
- let total = tokens[0]
- for (let j = cut; j <= i; j++) total += tokens[j]
+ let total = tokens[0] + leadingUserTokens[i]
+ for (let j = cut; j < i; j++) total += tokens[j]
const cutBefore = cut
while (total > limit && cut < i) {