Skip to content

An AI-powered search engine with a generative UI

refactor: update model handling in streaming and chat route

Yoshiki Miurac793b035abb5
diff --git a/app/api/chat/route.ts b/app/api/chat/route.ts
index 63670cc9..478f8c72 100644
--- a/app/api/chat/route.ts
+++ b/app/api/chat/route.ts
@@ -1,13 +1,19 @@
import { createManualToolStreamResponse } from '@/lib/streaming/create-manual-tool-stream'
import { createToolCallingStreamResponse } from '@/lib/streaming/create-tool-calling-stream'
import { Model } from '@/lib/types/models'
-import { createModelId } from '@/lib/utils'
-import { isProviderEnabled, isToolCallSupported } from '@/lib/utils/registry'
+import { isProviderEnabled } from '@/lib/utils/registry'
import { cookies } from 'next/headers'
export const maxDuration = 30
-const DEFAULT_MODEL = 'gpt-4o-mini'
+const DEFAULT_MODEL: Model = {
+ id: 'gpt-4o-mini',
+ name: 'GPT-4o mini',
+ provider: 'OpenAI',
+ providerId: 'openai',
+ enabled: true,
+ toolCallType: 'native'
+}
export async function POST(req: Request) {
try {
@@ -25,39 +31,42 @@ export async function POST(req: Request) {
const cookieStore = await cookies()
const modelJson = cookieStore.get('selectedModel')?.value
const searchMode = cookieStore.get('search-mode')?.value === 'true'
-
- let modelId = DEFAULT_MODEL
- let provider = 'openai'
-
+
+ let selectedModel = DEFAULT_MODEL
+
if (modelJson) {
try {
- const selectedModel = JSON.parse(modelJson) as Model
- modelId = createModelId(selectedModel)
- provider = selectedModel.providerId
+ selectedModel = JSON.parse(modelJson) as Model
} catch (e) {
console.error('Failed to parse selected model:', e)
}
}
- if (!isProviderEnabled(provider)) {
- return new Response(`Selected provider is not enabled ${provider}`, {
- status: 404,
- statusText: 'Not Found'
- })
+ if (
+ !isProviderEnabled(selectedModel.providerId) ||
+ selectedModel.enabled === false
+ ) {
+ return new Response(
+ `Selected provider is not enabled ${selectedModel.providerId}`,
+ {
+ status: 404,
+ statusText: 'Not Found'
+ }
+ )
}
- const supportsToolCalling = isToolCallSupported(modelId)
+ const supportsToolCalling = selectedModel.toolCallType === 'native'
return supportsToolCalling
? createToolCallingStreamResponse({
messages,
- model: modelId,
+ model: selectedModel,
chatId,
searchMode
})
: createManualToolStreamResponse({
messages,
- model: modelId,
+ model: selectedModel,
chatId,
searchMode
})
diff --git a/lib/streaming/create-manual-tool-stream.ts b/lib/streaming/create-manual-tool-stream.ts
index 94e10194..0e656b77 100644
--- a/lib/streaming/create-manual-tool-stream.ts
+++ b/lib/streaming/create-manual-tool-stream.ts
@@ -16,6 +16,11 @@ export function createManualToolStreamResponse(config: BaseStreamConfig) {
return createDataStreamResponse({
execute: async (dataStream: DataStreamWriter) => {
const { messages, model, chatId, searchMode } = config
+ const modelId = `${model.providerId}:${model.id}`
+ let toolCallModelId = model.toolCallModel
+ ? `${model.providerId}:${model.toolCallModel}`
+ : modelId
+
try {
const coreMessages = convertToCoreMessages(messages)
const truncatedMessages = truncateMessages(
@@ -27,13 +32,13 @@ export function createManualToolStreamResponse(config: BaseStreamConfig) {
await executeToolCall(
truncatedMessages,
dataStream,
- model,
+ toolCallModelId,
searchMode
)
const researcherConfig = manualResearcher({
messages: [...truncatedMessages, ...toolCallMessages],
- model,
+ model: modelId,
isSearchEnabled: searchMode
})
@@ -61,7 +66,7 @@ export function createManualToolStreamResponse(config: BaseStreamConfig) {
await handleStreamFinish({
responseMessages: result.response.messages,
originalMessages: messages,
- model,
+ model: modelId,
chatId,
dataStream,
skipRelatedQuestions: true,
diff --git a/lib/streaming/create-tool-calling-stream.ts b/lib/streaming/create-tool-calling-stream.ts
index 558c09ed..30e4de7d 100644
--- a/lib/streaming/create-tool-calling-stream.ts
+++ b/lib/streaming/create-tool-calling-stream.ts
@@ -14,6 +14,7 @@ export function createToolCallingStreamResponse(config: BaseStreamConfig) {
return createDataStreamResponse({
execute: async (dataStream: DataStreamWriter) => {
const { messages, model, chatId, searchMode } = config
+ const modelId = `${model.providerId}:${model.id}`
try {
const coreMessages = convertToCoreMessages(messages)
@@ -24,7 +25,7 @@ export function createToolCallingStreamResponse(config: BaseStreamConfig) {
let researcherConfig = await researcher({
messages: truncatedMessages,
- model,
+ model: modelId,
searchMode
})
@@ -34,10 +35,10 @@ export function createToolCallingStreamResponse(config: BaseStreamConfig) {
await handleStreamFinish({
responseMessages: result.response.messages,
originalMessages: messages,
- model,
+ model: modelId,
chatId,
dataStream,
- skipRelatedQuestions: isReasoningModel(model)
+ skipRelatedQuestions: isReasoningModel(modelId)
})
}
})
diff --git a/lib/streaming/tool-execution.ts b/lib/streaming/tool-execution.ts
index b7b70f04..eeb1a1b4 100644
--- a/lib/streaming/tool-execution.ts
+++ b/lib/streaming/tool-execution.ts
@@ -9,7 +9,7 @@ import { z } from 'zod'
import { searchSchema } from '../schema/search'
import { search } from '../tools/search'
import { ExtendedCoreMessage } from '../types'
-import { getToolCallModel } from '../utils/registry'
+import { getModel } from '../utils/registry'
import { parseToolCallXml } from './parse-tool-call'
interface ToolExecutionResult {
@@ -28,7 +28,6 @@ export async function executeToolCall(
return { toolCallDataAnnotation: null, toolCallMessages: [] }
}
- const toolCallModel = getToolCallModel(model)
// Convert Zod schema to string representation
const searchSchemaString = Object.entries(searchSchema.shape)
.map(([key, value]) => {
@@ -41,7 +40,7 @@ export async function executeToolCall(
// Generate tool selection using XML format
const toolSelectionResponse = await generateText({
- model: toolCallModel,
+ model: getModel(model),
system: `You are an intelligent assistant that analyzes conversations to select the most appropriate tools and their parameters.
You excel at understanding context to determine when and how to use available tools, including crafting effective search queries.
Current date: ${new Date().toISOString().split('T')[0]}
diff --git a/lib/streaming/types.ts b/lib/streaming/types.ts
index 7825f31b..eed1eb4b 100644
--- a/lib/streaming/types.ts
+++ b/lib/streaming/types.ts
@@ -1,8 +1,9 @@
import { Message } from 'ai'
+import { Model } from '../types/models'
export interface BaseStreamConfig {
messages: Message[]
- model: string
+ model: Model
chatId: string
searchMode: boolean
}
diff --git a/lib/utils/context-window.ts b/lib/utils/context-window.ts
index bedc8ca0..ae09a5d2 100644
--- a/lib/utils/context-window.ts
+++ b/lib/utils/context-window.ts
@@ -1,16 +1,17 @@
import { CoreMessage } from 'ai'
+import { Model } from '../types/models'
const DEFAULT_CONTEXT_WINDOW = 128_000
const DEFAULT_RESERVE_TOKENS = 30_000
-export function getMaxAllowedTokens(modelId: string): number {
+export function getMaxAllowedTokens(model: Model): number {
let contextWindow: number
let reserveTokens: number
- if (modelId.includes('deepseek')) {
+ if (model.id.includes('deepseek')) {
contextWindow = 64_000
reserveTokens = 27_000
- } else if (modelId.includes('claude')) {
+ } else if (model.id.includes('claude')) {
contextWindow = 200_000
reserveTokens = 40_000
} else {
@@ -30,21 +31,15 @@ export function truncateMessages(
for (let i = messages.length - 1; i >= 0; i--) {
const message = messages[i]
- const messageTokens = message.content?.length || 0
+ const messageTokens = message.content.length / 4 // rough estimation
if (totalTokens + messageTokens <= maxTokens) {
- tempMessages.push(message)
+ tempMessages.unshift(message)
totalTokens += messageTokens
} else {
break
}
}
- const orderedMessages = tempMessages.reverse()
-
- while (orderedMessages.length > 0 && orderedMessages[0].role !== 'user') {
- orderedMessages.shift()
- }
-
- return orderedMessages
+ return tempMessages
}