Skip to content

An AI-powered search engine with a generative UI

feat: implement message truncation based on model token limits and add utility functions

Yoshiki Miura2709e49af068
diff --git a/lib/streaming/create-manual-tool-stream.ts b/lib/streaming/create-manual-tool-stream.ts
index 3eae4a50..90f59a23 100644
--- a/lib/streaming/create-manual-tool-stream.ts
+++ b/lib/streaming/create-manual-tool-stream.ts
@@ -7,6 +7,7 @@ import {
} from 'ai'
import { manualResearcher } from '../agents/manual-researcher'
import { ExtendedCoreMessage } from '../types'
+import { getMaxAllowedTokens, truncateMessages } from '../utils/context-window'
import { handleStreamFinish } from './handle-stream-finish'
import { executeToolCall } from './tool-execution'
import { BaseStreamConfig } from './types'
@@ -17,12 +18,16 @@ export function createManualToolStreamResponse(config: BaseStreamConfig) {
const { messages, model, chatId } = config
try {
const coreMessages = convertToCoreMessages(messages)
+ const truncatedMessages = truncateMessages(
+ coreMessages,
+ getMaxAllowedTokens(model)
+ )
const { toolCallDataAnnotation, toolCallMessages } =
- await executeToolCall(coreMessages, dataStream)
+ await executeToolCall(truncatedMessages, dataStream)
const researcherConfig = manualResearcher({
- messages: [...coreMessages, ...toolCallMessages],
+ messages: [...truncatedMessages, ...toolCallMessages],
model
})
@@ -46,7 +51,7 @@ export function createManualToolStreamResponse(config: BaseStreamConfig) {
model,
chatId,
dataStream,
- skipRelatedQuestions: false,
+ skipRelatedQuestions: true,
annotations
})
}
diff --git a/lib/streaming/create-tool-calling-stream.ts b/lib/streaming/create-tool-calling-stream.ts
index fa4bec7b..80bb8547 100644
--- a/lib/streaming/create-tool-calling-stream.ts
+++ b/lib/streaming/create-tool-calling-stream.ts
@@ -5,6 +5,7 @@ import {
DataStreamWriter,
streamText
} from 'ai'
+import { getMaxAllowedTokens, truncateMessages } from '../utils/context-window'
import { handleStreamFinish } from './handle-stream-finish'
import { BaseStreamConfig } from './types'
@@ -15,9 +16,13 @@ export function createToolCallingStreamResponse(config: BaseStreamConfig) {
try {
const coreMessages = convertToCoreMessages(messages)
+ const truncatedMessages = truncateMessages(
+ coreMessages,
+ getMaxAllowedTokens(model)
+ )
let researcherConfig = await researcher({
- messages: coreMessages,
+ messages: truncatedMessages,
model
})
diff --git a/lib/utils/context-window.ts b/lib/utils/context-window.ts
new file mode 100644
index 00000000..6809f98d
--- /dev/null
+++ b/lib/utils/context-window.ts
@@ -0,0 +1,50 @@
+import { CoreMessage } from 'ai'
+
+const DEFAULT_CONTEXT_WINDOW = 128_000
+const DEFAULT_RESERVE_TOKENS = 30_000
+
+export function getMaxAllowedTokens(modelId: string): number {
+ let contextWindow: number
+ let reserveTokens: number
+
+ if (modelId.includes('deepseek')) {
+ contextWindow = 64_000
+ reserveTokens = 27_000
+ } else if (modelId.includes('claude')) {
+ contextWindow = 200_000
+ reserveTokens = 40_000
+ } else {
+ contextWindow = DEFAULT_CONTEXT_WINDOW
+ reserveTokens = DEFAULT_RESERVE_TOKENS
+ }
+
+ return contextWindow - reserveTokens
+}
+
+export function truncateMessages(
+ messages: CoreMessage[],
+ maxTokens: number,
+ preserveSystemMessage = true
+): CoreMessage[] {
+ let totalTokens = 0
+ const truncatedMessages: CoreMessage[] = []
+
+ if (preserveSystemMessage && messages.length > 0) {
+ truncatedMessages.push(messages[0])
+ totalTokens += messages[0].content?.length || 0
+ }
+
+ for (let i = messages.length - 1; i >= (preserveSystemMessage ? 1 : 0); i--) {
+ const message = messages[i]
+ const messageTokens = message.content?.length || 0
+
+ if (totalTokens + messageTokens <= maxTokens) {
+ truncatedMessages.unshift(message)
+ totalTokens += messageTokens
+ } else {
+ break
+ }
+ }
+
+ return truncatedMessages
+}