An AI-powered search engine with a generative UI
TypeScript generative-aigenerative-uinextjsollamareactsearxngshadcn-uitailwindcsstavilytypescriptupstashvercel-ai-sdk
feat: implement context window management for LLMs
Yoshiki Miura77c4d52ecd5c
diff --git a/bun.lock b/bun.lockindex 2bc4fdf2..a47ad2a6 100644--- a/bun.lock+++ b/bun.lock@@ -47,6 +47,7 @@"drizzle-orm": "^0.44.3","embla-carousel-react": "^8.0.0","exa-js": "^1.0.12",+ "js-tiktoken": "^1.0.20","jsdom": "^26.1.0","katex": "^0.16.10","lucide-react": "^0.507.0",@@ -884,6 +885,8 @@"balanced-match": ["[email protected]", "", {}, "sha512-3oSeUO0TMV67hN1AmbXsK4yaqU7tjiHlbxRDZOpH0KW9+CeX4bRAaX0Anxt0tx2MrpRpWwQaPwIlISEJhYU5Pw=="],+ "base64-js": ["[email protected]", "", {}, "sha512-AKpaYlHn8t4SVbOHCy+b5+KKgvR4vrsD8vbvrbiQJps7fKDTkjkDry6ji0rUJjC0kzbNePLwzxq8iypo41qeWA=="],+"binary-extensions": ["[email protected]", "", {}, "sha512-Ceh+7ox5qe7LJuLHoY0feh3pHuUDHAcRUeyL2VYghZwfpkNIy/+8Ocg0a3UuSoYzavmylwuLWQOf3hl0jjMMIw=="],"boolbase": ["[email protected]", "", {}, "sha512-JZOSA7Mo9sNGB8+UjSgzdLtokWAky1zbztM3WRLCbZ70/3cTANmQmOdR7y2g+J0e2WXywy1yS468tY+IruqEww=="],@@ -1332,6 +1335,8 @@"jose": ["[email protected]", "", {}, "sha512-1vUQX+IdDMVPj4k8kOxgUqlcK518yluMuGZwqlr44FS1ppZB/5GWh4rZG89erpOBOJjU/OBsnCVFfapsRz6nEA=="],+ "js-tiktoken": ["[email protected]", "", { "dependencies": { "base64-js": "^1.5.1" } }, "sha512-Xlaqhhs8VfCd6Sh7a1cFkZHQbYTLCwVJJWiHVxBYzLPxW0XsoxBy1hitmjkdIjD3Aon5BXLHFwU5O8WUx6HH+A=="],+"js-tokens": ["[email protected]", "", {}, "sha512-RdJUflcE3cUzKiMqQgsCu06FPu9UdIJO0beYbPhHN4k6apgJtifcoCtT9bcxOpYBtpD2kCM6Sbzg4CausW/PKQ=="],"js-yaml": ["[email protected]", "", { "dependencies": { "argparse": "^2.0.1" }, "bin": { "js-yaml": "bin/js-yaml.js" } }, "sha512-wpxZs9NoxZaJESJGIZTyDEaYpl0FKSA+FB9aJiyemKhMwkxQg63h4T1KJgUGHpTqPDNRcmmYLugrRjJlBtWvRA=="],diff --git a/lib/streaming/create-chat-stream-response.ts b/lib/streaming/create-chat-stream-response.tsindex 31769cb5..9083a2e6 100644--- a/lib/streaming/create-chat-stream-response.ts+++ b/lib/streaming/create-chat-stream-response.ts@@ -17,6 +17,11 @@ import {import { generateRelatedQuestions } from '../agents/generate-related-questions'import { generateChatTitle } from '../agents/title-generator'import { generateId } from '../db/schema'+import {+ getMaxAllowedTokens,+ shouldTruncateMessages,+ truncateMessages+} from '../utils/context-window'import {getTextFromParts,hasToolCalls,@@ -155,9 +160,24 @@ export async function createChatStreamResponse(searchMode})+ // Convert to model messages and apply context window management+ let modelMessages = convertToModelMessages(messagesToModel)++ if (shouldTruncateMessages(modelMessages, model)) {+ const maxTokens = getMaxAllowedTokens(model)+ const originalCount = modelMessages.length+ modelMessages = truncateMessages(modelMessages, maxTokens, model.id)++ if (process.env.NODE_ENV === 'development') {+ console.log(+ `Context window limit reached. Truncating from ${originalCount} to ${modelMessages.length} messages`+ )+ }+ }+// Stream with the research agentconst researchResult = researchAgent.stream({- messages: convertToModelMessages(messagesToModel)+ messages: modelMessages})// Store messages for mergingdiff --git a/lib/utils/__tests__/context-window.test.ts b/lib/utils/__tests__/context-window.test.tsnew file mode 100644index 00000000..4b17ec45--- /dev/null+++ b/lib/utils/__tests__/context-window.test.ts@@ -0,0 +1,206 @@+import { ModelMessage } from 'ai'+import { describe, expect, test } from 'vitest'++import { Model } from '@/lib/types/models'++import {+ getMaxAllowedTokens,+ shouldTruncateMessages,+ truncateMessages+} from '../context-window'++describe('context-window', () => {+ const mockModel: Model = {+ id: 'gpt-4o-mini',+ name: 'GPT-4o mini',+ provider: 'OpenAI',+ providerId: 'openai',+ enabled: true,+ toolCallType: 'native'+ }++ const createMessage = (+ role: 'user' | 'assistant',+ content: string+ ): ModelMessage => ({+ role,+ content+ })++ describe('getMaxAllowedTokens', () => {+ test('calculates max tokens correctly for known model', () => {+ const maxTokens = getMaxAllowedTokens(mockModel)+ // Expected: (128000 - 16384) - (128000 * 0.1) = 111616 - 12800 = 98816+ expect(maxTokens).toBe(98816)+ })++ test('uses default values for unknown model', () => {+ const unknownModel: Model = {+ ...mockModel,+ id: 'unknown-model'+ }+ const maxTokens = getMaxAllowedTokens(unknownModel)+ // Expected: (16384 - 4096) - (16384 * 0.1) = 12288 - 1638.4 = 10649.6 -> 10650+ expect(maxTokens).toBe(10650)+ })++ test('ensures minimum viable token count', () => {+ // This would need a model with very small context window to test+ // For now, verify the function returns at least 1000+ const maxTokens = getMaxAllowedTokens(mockModel)+ expect(maxTokens).toBeGreaterThanOrEqual(1000)+ })+ })++ describe('shouldTruncateMessages', () => {+ test('returns false for empty messages', () => {+ expect(shouldTruncateMessages([], mockModel)).toBe(false)+ })++ test('returns false when under limit', () => {+ const messages: ModelMessage[] = [+ createMessage('user', 'Hello'),+ createMessage('assistant', 'Hi there!')+ ]+ expect(shouldTruncateMessages(messages, mockModel)).toBe(false)+ })++ test('returns true when over limit', () => {+ // Create messages that exceed the token limit+ // mockModel (gpt-4o-mini) has 98816 max tokens+ const longText = 'This is a test message. '.repeat(1000) // ~6000 tokens per message+ const messages: ModelMessage[] = Array(20)+ .fill(null)+ .map(() => createMessage('user', longText)) // Total: ~120,000 tokens > 98,816 max tokens+ expect(shouldTruncateMessages(messages, mockModel)).toBe(true)+ })++ test('handles null/undefined messages gracefully', () => {+ expect(shouldTruncateMessages(null as any, mockModel)).toBe(false)+ expect(shouldTruncateMessages(undefined as any, mockModel)).toBe(false)+ })+ })++ describe('truncateMessages', () => {+ test('returns empty array for empty messages', () => {+ expect(truncateMessages([], 1000)).toEqual([])+ })++ test('returns empty array for invalid maxTokens', () => {+ const messages = [createMessage('user', 'Hello')]+ expect(truncateMessages(messages, 0)).toEqual([])+ expect(truncateMessages(messages, -100)).toEqual([])+ })++ test('returns all messages when under limit', () => {+ const messages: ModelMessage[] = [+ createMessage('user', 'Hello'),+ createMessage('assistant', 'Hi!'),+ createMessage('user', 'How are you?'),+ createMessage('assistant', 'I am fine!')+ ]+ const result = truncateMessages(messages, 10000)+ expect(result).toEqual(messages)+ })++ test('preserves first user message when possible', () => {+ const messages: ModelMessage[] = [+ createMessage('user', 'First important context'),+ createMessage('assistant', 'Response 1'),+ createMessage('user', 'Question 2'),+ createMessage('assistant', 'Response 2'),+ createMessage('user', 'Question 3'),+ createMessage('assistant', 'Response 3')+ ]++ const result = truncateMessages(messages, 100) // Very low limit+ expect(result.length).toBeGreaterThan(0)+ expect(result[0]).toEqual(messages[0]) // First user message preserved+ })++ test('removes assistant messages to keep user messages', () => {+ const messages: ModelMessage[] = [+ createMessage('user', 'Question 1'),+ createMessage('assistant', 'Very long response '.repeat(50)),+ createMessage('user', 'Question 2'),+ createMessage('assistant', 'Another long response '.repeat(50)),+ createMessage('user', 'Important last question')+ ]++ const result = truncateMessages(messages, 200)+ const userMessages = result.filter(m => m.role === 'user')+ expect(userMessages.length).toBeGreaterThan(0)+ expect(userMessages[userMessages.length - 1].content).toBe(+ 'Important last question'+ )+ })++ test('removes leading assistant messages when truncating', () => {+ // Create messages that will force truncation+ const longText = 'a'.repeat(1000) // ~250 tokens each+ const messages: ModelMessage[] = [+ createMessage('assistant', longText),+ createMessage('assistant', longText),+ createMessage('user', 'Hello'),+ createMessage('assistant', 'Hi'),+ createMessage('user', 'Last message')+ ]++ // Force truncation with low limit+ const result = truncateMessages(messages, 100)++ // After truncation, should prefer user messages+ expect(result.length).toBeGreaterThan(0)++ // The implementation removes leading non-user messages after truncation+ const hasUserMessage = result.some(m => m.role === 'user')+ expect(hasUserMessage).toBe(true)+ })++ test('handles messages with complex content types', () => {+ const messages: ModelMessage[] = [+ {+ role: 'user',+ content: [+ { type: 'text', text: 'Hello' },+ { type: 'text', text: 'World' }+ ]+ },+ {+ role: 'assistant',+ content: 'Response'+ }+ ]++ const result = truncateMessages(messages, 1000)+ expect(result.length).toBeGreaterThan(0)+ })++ test('handles undefined content gracefully', () => {+ const messages: ModelMessage[] = [+ { role: 'user', content: '' },+ { role: 'assistant', content: 'Response' }+ ]++ const result = truncateMessages(messages, 1000)+ expect(result).toBeDefined()+ expect(Array.isArray(result)).toBe(true)+ })+ })++ describe('truncation with model ID', () => {+ test('uses tiktoken when model ID is provided', () => {+ const messages: ModelMessage[] = [+ createMessage('user', 'Test message for token counting')+ ]++ // With model ID - should use tiktoken+ const resultWithModel = truncateMessages(messages, 1000, 'gpt-4o-mini')+ expect(resultWithModel).toBeDefined()++ // Without model ID - should use fallback+ const resultWithoutModel = truncateMessages(messages, 1000)+ expect(resultWithoutModel).toBeDefined()+ })+ })+})diff --git a/lib/utils/context-window.ts b/lib/utils/context-window.tsindex ef817377..fb794c31 100644--- a/lib/utils/context-window.ts+++ b/lib/utils/context-window.ts@@ -1,52 +1,302 @@import { ModelMessage } from 'ai'+import { getEncoding, type TiktokenEncoding } from 'js-tiktoken'import { Model } from '../types/models'-const DEFAULT_CONTEXT_WINDOW = 128_000-const DEFAULT_RESERVE_TOKENS = 30_000+interface ModelContextInfo {+ contextWindow: number+ outputTokens: number+}++// Model-specific context window configurations+const MODEL_CONTEXT_WINDOWS: Record<string, ModelContextInfo> = {+ // OpenAI Models+ 'gpt-4.1': { contextWindow: 128000, outputTokens: 16384 },+ 'gpt-4.1-mini': { contextWindow: 128000, outputTokens: 16384 },+ 'gpt-4.1-nano': { contextWindow: 128000, outputTokens: 16384 },+ 'gpt-4o-mini': { contextWindow: 128000, outputTokens: 16384 },++ // Anthropic Models+ 'claude-opus-4': { contextWindow: 680000, outputTokens: 8192 },+ 'claude-sonnet-4': { contextWindow: 680000, outputTokens: 8192 },+ 'claude-3-7-sonnet': { contextWindow: 200000, outputTokens: 8192 },+ 'claude-3-7-sonnet-20250219': { contextWindow: 200000, outputTokens: 8192 },+ 'claude-3-5-haiku-20241022': { contextWindow: 200000, outputTokens: 8192 },++ // Google Models+ 'gemini-2.5-flash': { contextWindow: 1048576, outputTokens: 65536 },+ 'gemini-2.5-pro': { contextWindow: 1048576, outputTokens: 65536 },++ // xAI Models+ 'grok-4-0709': { contextWindow: 256000, outputTokens: 8192 },+ 'grok-3': { contextWindow: 131072, outputTokens: 8192 },+ 'grok-3-mini': { contextWindow: 131072, outputTokens: 8192 }+}++// Default values for unknown models+const DEFAULT_CONTEXT_WINDOW = 16384+const DEFAULT_OUTPUT_TOKENS = 4096++// Safety buffer percentage (reserved for system prompts and formatting)+const SAFETY_BUFFER_RATIO = 0.1++// Cache for tiktoken encoders+const encoderCache = new Map<string, any>()++// Mapping of our model IDs to tiktoken encoding names+// js-tiktoken supports 'cl100k_base' (for GPT-4), 'p50k_base', 'r50k_base'+const MODEL_TO_ENCODING: Record<string, TiktokenEncoding> = {+ 'gpt-4.1': 'cl100k_base',+ 'gpt-4.1-mini': 'cl100k_base',+ 'gpt-4.1-nano': 'cl100k_base',+ 'gpt-4o-mini': 'cl100k_base',+ 'claude-opus-4': 'cl100k_base', // Use GPT-4 tokenizer as approximation for Claude+ 'claude-sonnet-4': 'cl100k_base',+ 'claude-3-7-sonnet': 'cl100k_base',+ 'claude-3-7-sonnet-20250219': 'cl100k_base',+ 'claude-3-5-haiku-20241022': 'cl100k_base',+ 'gemini-2.5-flash': 'cl100k_base', // Use GPT-4 tokenizer as approximation for Gemini+ 'gemini-2.5-pro': 'cl100k_base',+ 'grok-4-0709': 'cl100k_base', // Use GPT-4 tokenizer as approximation for Grok+ 'grok-3': 'cl100k_base',+ 'grok-3-mini': 'cl100k_base'+}++/**+ * Get model-specific context window information+ */+function getModelContextInfo(modelId: string): ModelContextInfo {+ // Direct lookup only+ return (+ MODEL_CONTEXT_WINDOWS[modelId] || {+ contextWindow: DEFAULT_CONTEXT_WINDOW,+ outputTokens: DEFAULT_OUTPUT_TOKENS+ }+ )+}+/**+ * Calculate the maximum allowed tokens for input+ */export function getMaxAllowedTokens(model: Model): number {- let contextWindow: number- let reserveTokens: number-- if (model.id.includes('deepseek')) {- contextWindow = 64_000- reserveTokens = 27_000- } else if (model.id.includes('claude')) {- contextWindow = 200_000- reserveTokens = 40_000- } else {- contextWindow = DEFAULT_CONTEXT_WINDOW- reserveTokens = DEFAULT_RESERVE_TOKENS+ const { contextWindow, outputTokens } = getModelContextInfo(model.id)++ // Calculate available tokens for input+ let availableTokens = contextWindow - outputTokens++ // Apply safety buffer+ const safetyBuffer = Math.floor(contextWindow * SAFETY_BUFFER_RATIO)+ availableTokens -= safetyBuffer++ // Ensure minimum viable token count+ return Math.max(availableTokens, 1000)+}++/**+ * Extract text content from various message content types+ */+function extractTextContent(content: ModelMessage['content']): string {+ if (!content) return ''++ // Handle string content+ if (typeof content === 'string') {+ return content}- return contextWindow - reserveTokens+ // Handle array of parts+ if (Array.isArray(content)) {+ return content+ .map(part => {+ if ('text' in part) {+ return part.text+ }+ return ''+ })+ .join(' ')+ }++ // Handle other content types+ return ''}+/**+ * Get or create encoder for a model+ */+function getEncoder(modelId: string) {+ try {+ const encodingName: TiktokenEncoding =+ MODEL_TO_ENCODING[modelId] || 'cl100k_base'++ if (!encoderCache.has(encodingName)) {+ const encoder = getEncoding(encodingName)+ encoderCache.set(encodingName, encoder)+ }++ return encoderCache.get(encodingName)+ } catch (error) {+ if (process.env.NODE_ENV === 'development') {+ console.warn(+ `Failed to load tokenizer for model ${modelId}, falling back to estimation`,+ error+ )+ }+ return null+ }+}++/**+ * Estimate token count for a message+ * Uses tiktoken for accurate counting when available+ */+function estimateTokenCount(+ content: ModelMessage['content'],+ modelId?: string+): number {+ const text = extractTextContent(content)+ if (!text) return 0++ // Try to use tiktoken for accurate counting+ if (modelId) {+ const encoder = getEncoder(modelId)+ if (encoder) {+ try {+ const tokens = encoder.encode(text)+ const tokenCount = tokens.length+ const overhead = 4 // Message formatting tokens+ return tokenCount + overhead+ } catch (error) {+ if (process.env.NODE_ENV === 'development') {+ console.warn(+ 'Failed to encode text with tiktoken, falling back to estimation',+ error+ )+ }+ }+ }+ }++ // Fallback: Rough approximation+ // ~4 characters per token for English, adjust for other languages+ const baseCount = Math.ceil(text.length / 4)+ const overhead = 4 // Message formatting tokens++ return baseCount + overhead+}++/**+ * Smart message truncation with priority for context preservation+ */export function truncateMessages(messages: ModelMessage[],- maxTokens: number+ maxTokens: number,+ modelId?: string): ModelMessage[] {- let totalTokens = 0- const tempMessages: ModelMessage[] = []+ // Input validation+ if (!messages || messages.length === 0) return []+ if (maxTokens <= 0) {+ console.error('Invalid maxTokens value:', maxTokens)+ return []+ }++ // Always try to keep the first user message (initial context)+ const firstUserIndex = messages.findIndex(m => m.role === 'user')+ const firstUserMessage = firstUserIndex >= 0 ? messages[firstUserIndex] : null++ // Calculate token counts for all messages+ const messageTokenCounts = messages.map(msg => ({+ message: msg,+ tokens: estimateTokenCount(msg.content, modelId)+ }))++ // Calculate total tokens+ const totalTokens = messageTokenCounts.reduce(+ (sum, item) => sum + item.tokens,+ 0+ )++ // If under limit, return all messages+ if (totalTokens <= maxTokens) {+ return messages+ }++ // Strategy: Keep first user message + as many recent messages as possible+ const result: ModelMessage[] = []+ let usedTokens = 0++ // Reserve space for first user message if it exists+ if (firstUserMessage) {+ const firstUserTokens = estimateTokenCount(+ firstUserMessage.content,+ modelId+ )+ if (firstUserTokens < maxTokens * 0.3) {+ // Don't let first message take more than 30%+ result.push(firstUserMessage)+ usedTokens += firstUserTokens+ }+ }++ // Add recent messages from the end+ const recentMessages: ModelMessage[] = []for (let i = messages.length - 1; i >= 0; i--) {- const message = messages[i]- const messageTokens = message.content?.length || 0+ const { message, tokens } = messageTokenCounts[i]++ // Skip if this is the first user message (already added)+ if (firstUserMessage && i === firstUserIndex) continue- if (totalTokens + messageTokens <= maxTokens) {- tempMessages.push(message)- totalTokens += messageTokens+ if (usedTokens + tokens <= maxTokens) {+ recentMessages.unshift(message)+ usedTokens += tokens} else {+ // Try to at least include the last user message if we haven't+ if (message.role === 'user' && recentMessages.length > 0) {+ // Remove oldest assistant messages to make room+ while (recentMessages.length > 0 && usedTokens + tokens > maxTokens) {+ const removed = recentMessages.shift()+ if (removed) {+ usedTokens -= estimateTokenCount(removed.content, modelId)+ }+ }+ if (usedTokens + tokens <= maxTokens) {+ recentMessages.unshift(message)+ usedTokens += tokens+ }+ }break}}- const orderedMessages = tempMessages.reverse()+ // Combine results, ensuring conversation flow+ if (firstUserMessage && result.length > 0) {+ // If we kept the first message, add recent ones+ result.push(...recentMessages)+ } else {+ // Otherwise, just use recent messages+ result.push(...recentMessages)+ }- while (orderedMessages.length > 0 && orderedMessages[0].role !== 'user') {- orderedMessages.shift()+ // Ensure the result starts with a user message+ while (result.length > 0 && result[0].role !== 'user') {+ result.shift()}- return orderedMessages+ return result+}++/**+ * Check if messages need truncation+ */+export function shouldTruncateMessages(+ messages: ModelMessage[],+ model: Model+): boolean {+ if (!messages || messages.length === 0) return false++ const maxTokens = getMaxAllowedTokens(model)+ const totalTokens = messages.reduce(+ (sum, msg) => sum + estimateTokenCount(msg.content, model.id),+ 0+ )+ return totalTokens > maxTokens}diff --git a/package.json b/package.jsonindex a4a3350f..01af1de1 100644--- a/package.json+++ b/package.json@@ -60,6 +60,7 @@"drizzle-orm": "^0.44.3","embla-carousel-react": "^8.0.0","exa-js": "^1.0.12",+ "js-tiktoken": "^1.0.20","jsdom": "^26.1.0","katex": "^0.16.10","lucide-react": "^0.507.0",diff --git a/scripts/chat-cli.ts b/scripts/chat-cli.tsindex f78e84db..44e95579 100755--- a/scripts/chat-cli.ts+++ b/scripts/chat-cli.ts@@ -175,16 +175,28 @@ class ChatApiTester {console.log('💬 Chat ID:', this.config.chatId)console.log('\n---\n')+ // Build cookie string+ let cookieString = ''+ if (cookies) {+ // If cookies from env exist, append our settings to them+ cookieString = cookies+ if (!cookieString.includes('selectedModel=')) {+ cookieString += `; selectedModel=${encodeURIComponent(JSON.stringify(this.config.selectedModel))}`+ }+ if (!cookieString.includes('search-mode=')) {+ cookieString += `; search-mode=${this.config.searchMode}`+ }+ } else {+ // If no cookies from env, just use our settings+ cookieString = [+ `selectedModel=${encodeURIComponent(JSON.stringify(this.config.selectedModel))}`,+ `search-mode=${this.config.searchMode}`+ ].join('; ')+ }+const headers = {'Content-Type': 'application/json',- Cookie:- cookies ||- [- `selectedModel=${encodeURIComponent(JSON.stringify(this.config.selectedModel))}`,- `search-mode=${this.config.searchMode}`- ]- .filter(Boolean)- .join('; ')+ Cookie: cookieString}// Only show headers in debug mode, without sensitive data@@ -333,9 +345,9 @@ function parseArgs(): Partial<ChatApiConfig> {const modelId = args[++i]// Map common model namesconst modelMap: Record<string, any> = {- 'gpt-4o': {- id: 'gpt-4o',- name: 'GPT-4o',+ 'gpt-4.1': {+ id: 'gpt-4.1',+ name: 'GPT-4.1',provider: 'OpenAI',providerId: 'openai',enabled: true,@@ -349,9 +361,17 @@ function parseArgs(): Partial<ChatApiConfig> {enabled: true,toolCallType: 'native'},- 'claude-3-5-sonnet': {- id: 'claude-3-5-sonnet-latest',- name: 'Claude 3.5 Sonnet',+ 'claude-4-0-sonnet': {+ id: 'claude-4-0-sonnet',+ name: 'Claude 4.0 Sonnet',+ provider: 'Anthropic',+ providerId: 'anthropic',+ enabled: true,+ toolCallType: 'native'+ },+ 'claude-3-5-haiku-20241022': {+ id: 'claude-3-5-haiku-20241022',+ name: 'Claude 3.5 Haiku',provider: 'Anthropic',providerId: 'anthropic',enabled: true,