An AI-powered search engine with a generative UI
TypeScript generative-aigenerative-uinextjsollamareactsearxngshadcn-uitailwindcsstavilytypescriptupstashvercel-ai-sdk
Improve chat API performance with in-memory caching and optimized database queries
Yoshiki Miurae2cfac659538
diff --git a/lib/actions/chat.ts b/lib/actions/chat.tsindex 1bb08461..945d50ed 100644--- a/lib/actions/chat.ts+++ b/lib/actions/chat.ts@@ -31,13 +31,8 @@ export async function getChat(chatId: string,requestingUserId?: string): Promise<(Chat & { messages: UIMessage[] }) | null> {- const chat = await dbActions.getChat(chatId, requestingUserId)- if (!chat) {- return null- }-- const messages = await dbActions.loadChat(chatId)- return { ...chat, messages }+ // Use optimized function that loads both in parallel+ return dbActions.loadChatWithMessages(chatId, requestingUserId)}/**diff --git a/lib/cache/memory-cache.ts b/lib/cache/memory-cache.tsnew file mode 100644index 00000000..360d18c4--- /dev/null+++ b/lib/cache/memory-cache.ts@@ -0,0 +1,199 @@+import { Chat } from '@/lib/db/schema'+import { UIMessage } from '@/lib/types/ai'++// LRU cache entry type+type CacheEntry<T> = {+ value: T+ expiry: number+ lastAccessed: number+}++// In-memory cache with TTL and LRU eviction+class MemoryCache<T> {+ private cache = new Map<string, CacheEntry<T>>()+ private ttl: number+ private maxEntries: number+ private stats = { hits: 0, misses: 0 }++ constructor(ttlSeconds: number = 60, maxEntries: number = 1000) {+ this.ttl = ttlSeconds * 1000 // Convert to milliseconds+ this.maxEntries = maxEntries+ }++ set(key: string, value: T): void {+ try {+ // If we're at capacity, evict LRU entry+ if (this.cache.size >= this.maxEntries) {+ this.evictLRU()+ }++ const now = Date.now()+ const expiry = now + this.ttl+ this.cache.set(key, { value, expiry, lastAccessed: now })+ } catch (error) {+ console.error('Cache set error:', error)+ // Fail silently, don't impact main flow+ }+ }++ private evictLRU(): void {+ let oldestKey: string | null = null+ let oldestTime = Infinity++ for (const [key, entry] of this.cache.entries()) {+ if (entry.lastAccessed < oldestTime) {+ oldestTime = entry.lastAccessed+ oldestKey = key+ }+ }++ if (oldestKey) {+ this.cache.delete(oldestKey)+ }+ }++ get(key: string): T | null {+ try {+ const item = this.cache.get(key)++ if (!item) {+ this.stats.misses+++ return null+ }++ const now = Date.now()+ if (now > item.expiry) {+ this.cache.delete(key)+ this.stats.misses+++ return null+ }++ // Update last accessed time+ item.lastAccessed = now+ this.stats.hits+++ return item.value+ } catch (error) {+ console.error('Cache get error:', error)+ this.stats.misses+++ return null+ }+ }++ delete(key: string): void {+ try {+ this.cache.delete(key)+ } catch (error) {+ console.error('Cache delete error:', error)+ }+ }++ deletePattern(pattern: string): void {+ try {+ // Delete all keys that start with the pattern+ for (const key of this.cache.keys()) {+ if (key.startsWith(pattern)) {+ this.cache.delete(key)+ }+ }+ } catch (error) {+ console.error('Cache deletePattern error:', error)+ }+ }++ clear(): void {+ try {+ this.cache.clear()+ } catch (error) {+ console.error('Cache clear error:', error)+ }+ }++ // Clean up expired entries periodically+ cleanup(): void {+ try {+ const now = Date.now()+ for (const [key, item] of this.cache.entries()) {+ if (now > item.expiry) {+ this.cache.delete(key)+ }+ }+ } catch (error) {+ console.error('Cache cleanup error:', error)+ }+ }++ // Get cache statistics+ getStats() {+ return {+ ...this.stats,+ size: this.cache.size,+ hitRate: this.stats.hits > 0 ? this.stats.hits / (this.stats.hits + this.stats.misses) : 0+ }+ }+}++// Get cache configuration from environment variables+const getCacheConfig = () => {+ const ttl = process.env.CHAT_CACHE_TTL+ const maxEntries = process.env.CHAT_CACHE_MAX_ENTRIES++ const ttlValue = ttl ? parseInt(ttl, 10) : 300 // Default: 5 minutes+ const maxEntriesValue = maxEntries ? parseInt(maxEntries, 10) : 1000 // Default: 1000 entries++ // Validate configuration+ const validTtl = Math.max(10, Math.min(3600, ttlValue)) // Min: 10s, Max: 1 hour+ const validMaxEntries = Math.max(100, Math.min(10000, maxEntriesValue)) // Min: 100, Max: 10000++ return { ttl: validTtl, maxEntries: validMaxEntries }+}++// Cache instances with configurable TTLs and size limits+const config = getCacheConfig()+export const chatCache = new MemoryCache<Chat & { messages: UIMessage[] }>(config.ttl, config.maxEntries)++// Store interval reference for cleanup+let cleanupInterval: NodeJS.Timeout | null = null++// Setup cleanup interval+if (typeof setInterval !== 'undefined') {+ cleanupInterval = setInterval(() => {+ chatCache.cleanup()++ // Log stats periodically in development+ if (process.env.NODE_ENV === 'development') {+ const stats = chatCache.getStats()+ console.log(`Cache stats - Size: ${stats.size}, Hit rate: ${(stats.hitRate * 100).toFixed(2)}%`)+ }+ }, 60000) // Run cleanup every minute+}++// Cleanup function for graceful shutdown+export const cleanupCacheInterval = () => {+ if (cleanupInterval) {+ clearInterval(cleanupInterval)+ cleanupInterval = null+ }+ chatCache.clear()+}++// Setup graceful shutdown handlers+if (typeof process !== 'undefined' && process.on) {+ // Graceful shutdown on SIGINT (Ctrl+C)+ process.on('SIGINT', () => {+ console.log('\nGracefully shutting down cache...')+ cleanupCacheInterval()+ process.exit(0)+ })++ // Graceful shutdown on SIGTERM+ process.on('SIGTERM', () => {+ console.log('\nGracefully shutting down cache...')+ cleanupCacheInterval()+ process.exit(0)+ })++ // Cleanup before exit+ process.on('beforeExit', () => {+ cleanupCacheInterval()+ })+}\ No newline at end of filediff --git a/lib/db/actions.ts b/lib/db/actions.tsindex 071fc834..e00820ef 100644--- a/lib/db/actions.ts+++ b/lib/db/actions.ts@@ -2,6 +2,7 @@import { and, asc, desc, eq, gt, inArray } from 'drizzle-orm'+import { chatCache } from '@/lib/cache/memory-cache'import type { UIMessage } from '@/lib/types/ai'import type { PersistableUIMessage } from '@/lib/types/message-persistence'import {@@ -38,6 +39,9 @@ export async function createChat({}).returning()+ // Invalidate cache for this chat+ chatCache.deletePattern(`${id}-`)+return chat}@@ -76,7 +80,7 @@ export async function getChat(export async function upsertMessage(message: PersistableUIMessage & { chatId: string }): Promise<Message> {- return await db.transaction(async tx => {+ const result = await db.transaction(async tx => {// 1. Insert or update the messageconst messageData = mapUIMessageToDBMessage(message)const [dbMessage] = await tx@@ -101,6 +105,14 @@ export async function upsertMessage(return dbMessage})++ // Invalidate cache after successful transaction+ // Using setTimeout to ensure this happens after transaction commit+ setTimeout(() => {+ chatCache.deletePattern(`${message.chatId}-`)+ }, 0)++ return result}/**@@ -122,6 +134,53 @@ export async function loadChat(chatId: string): Promise<UIMessage[]> {return result.map(msg => buildUIMessageFromDB(msg, msg.parts))}+/**+ * Load chat with messages in a single query (optimized)+ */+export async function loadChatWithMessages(+ chatId: string,+ userId?: string+): Promise<(Chat & { messages: UIMessage[] }) | null> {+ // Don't check cache yet - need to verify permissions first++ // Get chat and messages in parallel+ const [chatResult, messagesResult] = await Promise.all([+ db.select().from(chats).where(eq(chats.id, chatId)).limit(1),+ db.query.messages.findMany({+ where: eq(messages.chatId, chatId),+ with: {+ parts: {+ orderBy: [asc(parts.order)]+ }+ },+ orderBy: [asc(messages.createdAt)]+ })+ ])++ const chat = chatResult[0]+ if (!chat) {+ return null+ }++ // Permission check+ if (chat.visibility === 'private' && (!userId || chat.userId !== userId)) {+ return null+ }++ // Now check cache with visibility included in key+ const cacheKey = `${chatId}-${userId || 'anonymous'}-${chat.visibility}`+ const cached = chatCache.get(cacheKey)+ if (cached) {+ return cached+ }++ // Build result and cache it+ const uiMessages = messagesResult.map(msg => buildUIMessageFromDB(msg, msg.parts))+ const result = { ...chat, messages: uiMessages }+ chatCache.set(cacheKey, result)+ return result+}+/*** Delete messages after a specific message*/@@ -156,6 +215,9 @@ export async function deleteMessagesAfter(if (messageIds.length > 0) {// Delete messages (parts will be cascade deleted)await db.delete(messages).where(inArray(messages.id, messageIds))++ // Invalidate cache for this chat+ chatCache.deletePattern(`${chatId}-`)}return { count: messageIds.length }@@ -188,6 +250,9 @@ export async function deleteMessagesFromIndex(if (messageIds.length > 0) {await db.delete(messages).where(inArray(messages.id, messageIds))++ // Invalidate cache for this chat+ chatCache.deletePattern(`${chatId}-`)}return { count: messageIds.length }@@ -220,6 +285,9 @@ export async function deleteChat(// Delete the chat (messages and parts will cascade)await db.delete(chats).where(eq(chats.id, chatId))++ // Invalidate cache for this chat+ chatCache.deletePattern(`${chatId}-`)return { success: true }} catch (error) {@@ -247,6 +315,11 @@ export async function updateChatVisibility(.where(eq(chats.id, chatId)).returning()+ // Invalidate cache for this chat+ if (updatedChat) {+ chatCache.deletePattern(`${chatId}-`)+ }+return updatedChat}@@ -263,5 +336,10 @@ export async function updateChatTitle(.where(eq(chats.id, chatId)).returning()+ // Invalidate cache for this chat+ if (updatedChat) {+ chatCache.deletePattern(`${chatId}-`)+ }+return updatedChat || null}diff --git a/lib/streaming/create-chat-stream-response.ts b/lib/streaming/create-chat-stream-response.tsindex 02c212c6..9dc2be1f 100644--- a/lib/streaming/create-chat-stream-response.ts+++ b/lib/streaming/create-chat-stream-response.ts@@ -22,6 +22,7 @@ import {hasToolCalls,mergeUIMessages} from '../utils/message-utils'+import { retryDatabaseOperation } from '../utils/retry'import { BaseStreamConfig } from './types'@@ -43,11 +44,11 @@ export async function createChatStreamResponse(})}- // Fetch chat data for authorization check- const chat = await getChatAction(chatId, userId)+ // Fetch chat data for authorization check and cache it+ let initialChat = await getChatAction(chatId, userId)// Authorization check: if chat exists, it must belong to the user- if (chat && chat.userId !== userId) {+ if (initialChat && initialChat.userId !== userId) {return new Response('You are not allowed to access this chat', {status: 403,statusText: 'Forbidden'@@ -62,8 +63,8 @@ export async function createChatStreamResponse(if (trigger === 'regenerate-assistant-message' && messageId) {// Handle regeneration- // Find the message to regenerate from- const currentChat = await getChatAction(chatId, userId)+ // Use cached chat data or fetch if not available+ const currentChat = initialChat || await getChatAction(chatId, userId)if (!currentChat || !currentChat.messages.length) {throw new Error('No messages found')}@@ -136,7 +137,7 @@ export async function createChatStreamResponse(}// If chat doesn't exist, create it with a temporary title- if (!chat) {+ if (!initialChat) {await createChat(chatId, DEFAULT_CHAT_TITLE)}@@ -192,21 +193,42 @@ export async function createChatStreamResponse(// If no tool calls (just answering), skip related questionsif (!hasToolCallsInMessage) {- // Save research message- await saveMessage(chatId, validResearchMessage)+ // Save research message and generate title in parallel+ const savePromises: Promise<any>[] = [saveMessage(chatId, validResearchMessage)]// Generate proper title after conversation starts- if (!chat && message) {+ if (!initialChat && message) {const userContent = getTextFromParts(message.parts)- const title = await generateChatTitle({- userMessageContent: userContent,- modelId- })- // Update chat title- const { updateChatTitle } = await import('../db/actions')- await updateChatTitle(chatId, title)+ savePromises.push(+ generateChatTitle({+ userMessageContent: userContent,+ modelId+ }).then(async title => {+ const { updateChatTitle } = await import('../db/actions')+ return updateChatTitle(chatId, title)+ })+ )}+ // Execute saves in background with exponential backoff retry+ Promise.all(savePromises).catch(async error => {+ console.error('Error saving message or title:', error)++ // Retry critical save operations with backoff+ try {+ for (const promise of savePromises) {+ await retryDatabaseOperation(+ () => promise,+ 'save message/title'+ )+ }+ } catch (retryError) {+ console.error('Failed to save after retries with backoff:', retryError)+ // Consider implementing alerting mechanism here+ // For now, we ensure the error is logged for monitoring+ }+ })+// Send a finish message to complete the streamwriter.write({ type: 'finish' })return@@ -252,28 +274,51 @@ export async function createChatStreamResponse(await relatedQuestionsPromise// Save the complete message after both agents finish+ const savePromises: Promise<any>[] = []+if (validResearchMessage && relatedQuestionsMessage) {const mergedMessage = mergeUIMessages(validResearchMessage,relatedQuestionsMessage)- await saveMessage(chatId, mergedMessage)+ savePromises.push(saveMessage(chatId, mergedMessage))} else if (validResearchMessage) {// Save research message only if related questions failed- await saveMessage(chatId, validResearchMessage)+ savePromises.push(saveMessage(chatId, validResearchMessage))}// Generate proper title after conversation starts- if (!chat && message) {+ if (!initialChat && message) {const userContent = getTextFromParts(message.parts)- const title = await generateChatTitle({- userMessageContent: userContent,- modelId- })- // Update chat title- const { updateChatTitle } = await import('../db/actions')- await updateChatTitle(chatId, title)+ savePromises.push(+ generateChatTitle({+ userMessageContent: userContent,+ modelId+ }).then(async title => {+ const { updateChatTitle } = await import('../db/actions')+ return updateChatTitle(chatId, title)+ })+ )}++ // Execute saves in background with exponential backoff retry+ Promise.all(savePromises).catch(async error => {+ console.error('Error saving message or title:', error)++ // Retry critical save operations with backoff+ try {+ for (const promise of savePromises) {+ await retryDatabaseOperation(+ () => promise,+ 'save message/title'+ )+ }+ } catch (retryError) {+ console.error('Failed to save after retries with backoff:', retryError)+ // Consider implementing alerting mechanism here+ // For now, we ensure the error is logged for monitoring+ }+ })} catch (error) {console.error('Error generating related questions:', error)// Save research message even if related questions faildiff --git a/lib/utils/retry.ts b/lib/utils/retry.tsnew file mode 100644index 00000000..5d8e0801--- /dev/null+++ b/lib/utils/retry.ts@@ -0,0 +1,66 @@+// Exponential backoff retry utility++export interface RetryOptions {+ maxRetries?: number+ initialDelayMs?: number+ maxDelayMs?: number+ backoffMultiplier?: number+ onRetry?: (error: any, attempt: number) => void+}++export async function retryWithBackoff<T>(+ fn: () => Promise<T>,+ options: RetryOptions = {}+): Promise<T> {+ const {+ maxRetries = 3,+ initialDelayMs = 100,+ maxDelayMs = 5000,+ backoffMultiplier = 2,+ onRetry+ } = options++ let lastError: any++ for (let attempt = 0; attempt <= maxRetries; attempt++) {+ try {+ return await fn()+ } catch (error) {+ lastError = error++ if (attempt === maxRetries) {+ throw error+ }++ // Calculate delay with exponential backoff+ const delay = Math.min(+ initialDelayMs * Math.pow(backoffMultiplier, attempt),+ maxDelayMs+ )++ if (onRetry) {+ onRetry(error, attempt + 1)+ }++ // Wait before retrying+ await new Promise(resolve => setTimeout(resolve, delay))+ }+ }++ throw lastError+}++// Specialized retry for database operations+export async function retryDatabaseOperation<T>(+ operation: () => Promise<T>,+ operationName: string+): Promise<T> {+ return retryWithBackoff(operation, {+ maxRetries: 2,+ initialDelayMs: 200,+ maxDelayMs: 2000,+ onRetry: (error, attempt) => {+ console.log(`Retrying ${operationName} (attempt ${attempt}):`, error.message)+ }+ })+}\ No newline at end of file