An AI-powered search engine with a generative UI
TypeScript generative-aigenerative-uinextjsollamareactsearxngshadcn-uitailwindcsstavilytypescriptupstashvercel-ai-sdk
feat: implement message truncation based on model token limits and add utility functions
Yoshiki Miura2709e49af068
diff --git a/lib/streaming/create-manual-tool-stream.ts b/lib/streaming/create-manual-tool-stream.tsindex 3eae4a50..90f59a23 100644--- a/lib/streaming/create-manual-tool-stream.ts+++ b/lib/streaming/create-manual-tool-stream.ts@@ -7,6 +7,7 @@ import {} from 'ai'import { manualResearcher } from '../agents/manual-researcher'import { ExtendedCoreMessage } from '../types'+import { getMaxAllowedTokens, truncateMessages } from '../utils/context-window'import { handleStreamFinish } from './handle-stream-finish'import { executeToolCall } from './tool-execution'import { BaseStreamConfig } from './types'@@ -17,12 +18,16 @@ export function createManualToolStreamResponse(config: BaseStreamConfig) {const { messages, model, chatId } = configtry {const coreMessages = convertToCoreMessages(messages)+ const truncatedMessages = truncateMessages(+ coreMessages,+ getMaxAllowedTokens(model)+ )const { toolCallDataAnnotation, toolCallMessages } =- await executeToolCall(coreMessages, dataStream)+ await executeToolCall(truncatedMessages, dataStream)const researcherConfig = manualResearcher({- messages: [...coreMessages, ...toolCallMessages],+ messages: [...truncatedMessages, ...toolCallMessages],model})@@ -46,7 +51,7 @@ export function createManualToolStreamResponse(config: BaseStreamConfig) {model,chatId,dataStream,- skipRelatedQuestions: false,+ skipRelatedQuestions: true,annotations})}diff --git a/lib/streaming/create-tool-calling-stream.ts b/lib/streaming/create-tool-calling-stream.tsindex fa4bec7b..80bb8547 100644--- a/lib/streaming/create-tool-calling-stream.ts+++ b/lib/streaming/create-tool-calling-stream.ts@@ -5,6 +5,7 @@ import {DataStreamWriter,streamText} from 'ai'+import { getMaxAllowedTokens, truncateMessages } from '../utils/context-window'import { handleStreamFinish } from './handle-stream-finish'import { BaseStreamConfig } from './types'@@ -15,9 +16,13 @@ export function createToolCallingStreamResponse(config: BaseStreamConfig) {try {const coreMessages = convertToCoreMessages(messages)+ const truncatedMessages = truncateMessages(+ coreMessages,+ getMaxAllowedTokens(model)+ )let researcherConfig = await researcher({- messages: coreMessages,+ messages: truncatedMessages,model})diff --git a/lib/utils/context-window.ts b/lib/utils/context-window.tsnew file mode 100644index 00000000..6809f98d--- /dev/null+++ b/lib/utils/context-window.ts@@ -0,0 +1,50 @@+import { CoreMessage } from 'ai'++const DEFAULT_CONTEXT_WINDOW = 128_000+const DEFAULT_RESERVE_TOKENS = 30_000++export function getMaxAllowedTokens(modelId: string): number {+ let contextWindow: number+ let reserveTokens: number++ if (modelId.includes('deepseek')) {+ contextWindow = 64_000+ reserveTokens = 27_000+ } else if (modelId.includes('claude')) {+ contextWindow = 200_000+ reserveTokens = 40_000+ } else {+ contextWindow = DEFAULT_CONTEXT_WINDOW+ reserveTokens = DEFAULT_RESERVE_TOKENS+ }++ return contextWindow - reserveTokens+}++export function truncateMessages(+ messages: CoreMessage[],+ maxTokens: number,+ preserveSystemMessage = true+): CoreMessage[] {+ let totalTokens = 0+ const truncatedMessages: CoreMessage[] = []++ if (preserveSystemMessage && messages.length > 0) {+ truncatedMessages.push(messages[0])+ totalTokens += messages[0].content?.length || 0+ }++ for (let i = messages.length - 1; i >= (preserveSystemMessage ? 1 : 0); i--) {+ const message = messages[i]+ const messageTokens = message.content?.length || 0++ if (totalTokens + messageTokens <= maxTokens) {+ truncatedMessages.unshift(message)+ totalTokens += messageTokens+ } else {+ break+ }+ }++ return truncatedMessages+}