Skip to content

An AI-powered search engine with a generative UI

feat: improve chat API performance with caching and optimized queries

Yoshiki Miura52a6a143ce55
diff --git a/lib/actions/chat.ts b/lib/actions/chat.ts
index 1bb08461..945d50ed 100644
--- a/lib/actions/chat.ts
+++ b/lib/actions/chat.ts
@@ -31,13 +31,8 @@ export async function getChat(
chatId: string,
requestingUserId?: string
): Promise<(Chat & { messages: UIMessage[] }) | null> {
- const chat = await dbActions.getChat(chatId, requestingUserId)
- if (!chat) {
- return null
- }
-
- const messages = await dbActions.loadChat(chatId)
- return { ...chat, messages }
+ // Use optimized function that loads both in parallel
+ return dbActions.loadChatWithMessages(chatId, requestingUserId)
}
/**
diff --git a/lib/cache/memory-cache.ts b/lib/cache/memory-cache.ts
new file mode 100644
index 00000000..360d18c4
--- /dev/null
+++ b/lib/cache/memory-cache.ts
@@ -0,0 +1,199 @@
+import { Chat } from '@/lib/db/schema'
+import { UIMessage } from '@/lib/types/ai'
+
+// LRU cache entry type
+type CacheEntry<T> = {
+ value: T
+ expiry: number
+ lastAccessed: number
+}
+
+// In-memory cache with TTL and LRU eviction
+class MemoryCache<T> {
+ private cache = new Map<string, CacheEntry<T>>()
+ private ttl: number
+ private maxEntries: number
+ private stats = { hits: 0, misses: 0 }
+
+ constructor(ttlSeconds: number = 60, maxEntries: number = 1000) {
+ this.ttl = ttlSeconds * 1000 // Convert to milliseconds
+ this.maxEntries = maxEntries
+ }
+
+ set(key: string, value: T): void {
+ try {
+ // If we're at capacity, evict LRU entry
+ if (this.cache.size >= this.maxEntries) {
+ this.evictLRU()
+ }
+
+ const now = Date.now()
+ const expiry = now + this.ttl
+ this.cache.set(key, { value, expiry, lastAccessed: now })
+ } catch (error) {
+ console.error('Cache set error:', error)
+ // Fail silently, don't impact main flow
+ }
+ }
+
+ private evictLRU(): void {
+ let oldestKey: string | null = null
+ let oldestTime = Infinity
+
+ for (const [key, entry] of this.cache.entries()) {
+ if (entry.lastAccessed < oldestTime) {
+ oldestTime = entry.lastAccessed
+ oldestKey = key
+ }
+ }
+
+ if (oldestKey) {
+ this.cache.delete(oldestKey)
+ }
+ }
+
+ get(key: string): T | null {
+ try {
+ const item = this.cache.get(key)
+
+ if (!item) {
+ this.stats.misses++
+ return null
+ }
+
+ const now = Date.now()
+ if (now > item.expiry) {
+ this.cache.delete(key)
+ this.stats.misses++
+ return null
+ }
+
+ // Update last accessed time
+ item.lastAccessed = now
+ this.stats.hits++
+ return item.value
+ } catch (error) {
+ console.error('Cache get error:', error)
+ this.stats.misses++
+ return null
+ }
+ }
+
+ delete(key: string): void {
+ try {
+ this.cache.delete(key)
+ } catch (error) {
+ console.error('Cache delete error:', error)
+ }
+ }
+
+ deletePattern(pattern: string): void {
+ try {
+ // Delete all keys that start with the pattern
+ for (const key of this.cache.keys()) {
+ if (key.startsWith(pattern)) {
+ this.cache.delete(key)
+ }
+ }
+ } catch (error) {
+ console.error('Cache deletePattern error:', error)
+ }
+ }
+
+ clear(): void {
+ try {
+ this.cache.clear()
+ } catch (error) {
+ console.error('Cache clear error:', error)
+ }
+ }
+
+ // Clean up expired entries periodically
+ cleanup(): void {
+ try {
+ const now = Date.now()
+ for (const [key, item] of this.cache.entries()) {
+ if (now > item.expiry) {
+ this.cache.delete(key)
+ }
+ }
+ } catch (error) {
+ console.error('Cache cleanup error:', error)
+ }
+ }
+
+ // Get cache statistics
+ getStats() {
+ return {
+ ...this.stats,
+ size: this.cache.size,
+ hitRate: this.stats.hits > 0 ? this.stats.hits / (this.stats.hits + this.stats.misses) : 0
+ }
+ }
+}
+
+// Get cache configuration from environment variables
+const getCacheConfig = () => {
+ const ttl = process.env.CHAT_CACHE_TTL
+ const maxEntries = process.env.CHAT_CACHE_MAX_ENTRIES
+
+ const ttlValue = ttl ? parseInt(ttl, 10) : 300 // Default: 5 minutes
+ const maxEntriesValue = maxEntries ? parseInt(maxEntries, 10) : 1000 // Default: 1000 entries
+
+ // Validate configuration
+ const validTtl = Math.max(10, Math.min(3600, ttlValue)) // Min: 10s, Max: 1 hour
+ const validMaxEntries = Math.max(100, Math.min(10000, maxEntriesValue)) // Min: 100, Max: 10000
+
+ return { ttl: validTtl, maxEntries: validMaxEntries }
+}
+
+// Cache instances with configurable TTLs and size limits
+const config = getCacheConfig()
+export const chatCache = new MemoryCache<Chat & { messages: UIMessage[] }>(config.ttl, config.maxEntries)
+
+// Store interval reference for cleanup
+let cleanupInterval: NodeJS.Timeout | null = null
+
+// Setup cleanup interval
+if (typeof setInterval !== 'undefined') {
+ cleanupInterval = setInterval(() => {
+ chatCache.cleanup()
+
+ // Log stats periodically in development
+ if (process.env.NODE_ENV === 'development') {
+ const stats = chatCache.getStats()
+ console.log(`Cache stats - Size: ${stats.size}, Hit rate: ${(stats.hitRate * 100).toFixed(2)}%`)
+ }
+ }, 60000) // Run cleanup every minute
+}
+
+// Cleanup function for graceful shutdown
+export const cleanupCacheInterval = () => {
+ if (cleanupInterval) {
+ clearInterval(cleanupInterval)
+ cleanupInterval = null
+ }
+ chatCache.clear()
+}
+
+// Setup graceful shutdown handlers
+if (typeof process !== 'undefined' && process.on) {
+ // Graceful shutdown on SIGINT (Ctrl+C)
+ process.on('SIGINT', () => {
+ console.log('\nGracefully shutting down cache...')
+ cleanupCacheInterval()
+ process.exit(0)
+ })
+
+ // Graceful shutdown on SIGTERM
+ process.on('SIGTERM', () => {
+ console.log('\nGracefully shutting down cache...')
+ cleanupCacheInterval()
+ process.exit(0)
+ })
+
+ // Cleanup before exit
+ process.on('beforeExit', () => {
+ cleanupCacheInterval()
+ })
+}
\ No newline at end of file
diff --git a/lib/db/actions.ts b/lib/db/actions.ts
index 071fc834..e00820ef 100644
--- a/lib/db/actions.ts
+++ b/lib/db/actions.ts
@@ -2,6 +2,7 @@
import { and, asc, desc, eq, gt, inArray } from 'drizzle-orm'
+import { chatCache } from '@/lib/cache/memory-cache'
import type { UIMessage } from '@/lib/types/ai'
import type { PersistableUIMessage } from '@/lib/types/message-persistence'
import {
@@ -38,6 +39,9 @@ export async function createChat({
})
.returning()
+ // Invalidate cache for this chat
+ chatCache.deletePattern(`${id}-`)
+
return chat
}
@@ -76,7 +80,7 @@ export async function getChat(
export async function upsertMessage(
message: PersistableUIMessage & { chatId: string }
): Promise<Message> {
- return await db.transaction(async tx => {
+ const result = await db.transaction(async tx => {
// 1. Insert or update the message
const messageData = mapUIMessageToDBMessage(message)
const [dbMessage] = await tx
@@ -101,6 +105,14 @@ export async function upsertMessage(
return dbMessage
})
+
+ // Invalidate cache after successful transaction
+ // Using setTimeout to ensure this happens after transaction commit
+ setTimeout(() => {
+ chatCache.deletePattern(`${message.chatId}-`)
+ }, 0)
+
+ return result
}
/**
@@ -122,6 +134,53 @@ export async function loadChat(chatId: string): Promise<UIMessage[]> {
return result.map(msg => buildUIMessageFromDB(msg, msg.parts))
}
+/**
+ * Load chat with messages in a single query (optimized)
+ */
+export async function loadChatWithMessages(
+ chatId: string,
+ userId?: string
+): Promise<(Chat & { messages: UIMessage[] }) | null> {
+ // Don't check cache yet - need to verify permissions first
+
+ // Get chat and messages in parallel
+ const [chatResult, messagesResult] = await Promise.all([
+ db.select().from(chats).where(eq(chats.id, chatId)).limit(1),
+ db.query.messages.findMany({
+ where: eq(messages.chatId, chatId),
+ with: {
+ parts: {
+ orderBy: [asc(parts.order)]
+ }
+ },
+ orderBy: [asc(messages.createdAt)]
+ })
+ ])
+
+ const chat = chatResult[0]
+ if (!chat) {
+ return null
+ }
+
+ // Permission check
+ if (chat.visibility === 'private' && (!userId || chat.userId !== userId)) {
+ return null
+ }
+
+ // Now check cache with visibility included in key
+ const cacheKey = `${chatId}-${userId || 'anonymous'}-${chat.visibility}`
+ const cached = chatCache.get(cacheKey)
+ if (cached) {
+ return cached
+ }
+
+ // Build result and cache it
+ const uiMessages = messagesResult.map(msg => buildUIMessageFromDB(msg, msg.parts))
+ const result = { ...chat, messages: uiMessages }
+ chatCache.set(cacheKey, result)
+ return result
+}
+
/**
* Delete messages after a specific message
*/
@@ -156,6 +215,9 @@ export async function deleteMessagesAfter(
if (messageIds.length > 0) {
// Delete messages (parts will be cascade deleted)
await db.delete(messages).where(inArray(messages.id, messageIds))
+
+ // Invalidate cache for this chat
+ chatCache.deletePattern(`${chatId}-`)
}
return { count: messageIds.length }
@@ -188,6 +250,9 @@ export async function deleteMessagesFromIndex(
if (messageIds.length > 0) {
await db.delete(messages).where(inArray(messages.id, messageIds))
+
+ // Invalidate cache for this chat
+ chatCache.deletePattern(`${chatId}-`)
}
return { count: messageIds.length }
@@ -220,6 +285,9 @@ export async function deleteChat(
// Delete the chat (messages and parts will cascade)
await db.delete(chats).where(eq(chats.id, chatId))
+
+ // Invalidate cache for this chat
+ chatCache.deletePattern(`${chatId}-`)
return { success: true }
} catch (error) {
@@ -247,6 +315,11 @@ export async function updateChatVisibility(
.where(eq(chats.id, chatId))
.returning()
+ // Invalidate cache for this chat
+ if (updatedChat) {
+ chatCache.deletePattern(`${chatId}-`)
+ }
+
return updatedChat
}
@@ -263,5 +336,10 @@ export async function updateChatTitle(
.where(eq(chats.id, chatId))
.returning()
+ // Invalidate cache for this chat
+ if (updatedChat) {
+ chatCache.deletePattern(`${chatId}-`)
+ }
+
return updatedChat || null
}
diff --git a/lib/streaming/create-chat-stream-response.ts b/lib/streaming/create-chat-stream-response.ts
index 02c212c6..9dc2be1f 100644
--- a/lib/streaming/create-chat-stream-response.ts
+++ b/lib/streaming/create-chat-stream-response.ts
@@ -22,6 +22,7 @@ import {
hasToolCalls,
mergeUIMessages
} from '../utils/message-utils'
+import { retryDatabaseOperation } from '../utils/retry'
import { BaseStreamConfig } from './types'
@@ -43,11 +44,11 @@ export async function createChatStreamResponse(
})
}
- // Fetch chat data for authorization check
- const chat = await getChatAction(chatId, userId)
+ // Fetch chat data for authorization check and cache it
+ let initialChat = await getChatAction(chatId, userId)
// Authorization check: if chat exists, it must belong to the user
- if (chat && chat.userId !== userId) {
+ if (initialChat && initialChat.userId !== userId) {
return new Response('You are not allowed to access this chat', {
status: 403,
statusText: 'Forbidden'
@@ -62,8 +63,8 @@ export async function createChatStreamResponse(
if (trigger === 'regenerate-assistant-message' && messageId) {
// Handle regeneration
- // Find the message to regenerate from
- const currentChat = await getChatAction(chatId, userId)
+ // Use cached chat data or fetch if not available
+ const currentChat = initialChat || await getChatAction(chatId, userId)
if (!currentChat || !currentChat.messages.length) {
throw new Error('No messages found')
}
@@ -136,7 +137,7 @@ export async function createChatStreamResponse(
}
// If chat doesn't exist, create it with a temporary title
- if (!chat) {
+ if (!initialChat) {
await createChat(chatId, DEFAULT_CHAT_TITLE)
}
@@ -192,21 +193,42 @@ export async function createChatStreamResponse(
// If no tool calls (just answering), skip related questions
if (!hasToolCallsInMessage) {
- // Save research message
- await saveMessage(chatId, validResearchMessage)
+ // Save research message and generate title in parallel
+ const savePromises: Promise<any>[] = [saveMessage(chatId, validResearchMessage)]
// Generate proper title after conversation starts
- if (!chat && message) {
+ if (!initialChat && message) {
const userContent = getTextFromParts(message.parts)
- const title = await generateChatTitle({
- userMessageContent: userContent,
- modelId
- })
- // Update chat title
- const { updateChatTitle } = await import('../db/actions')
- await updateChatTitle(chatId, title)
+ savePromises.push(
+ generateChatTitle({
+ userMessageContent: userContent,
+ modelId
+ }).then(async title => {
+ const { updateChatTitle } = await import('../db/actions')
+ return updateChatTitle(chatId, title)
+ })
+ )
}
+ // Execute saves in background with exponential backoff retry
+ Promise.all(savePromises).catch(async error => {
+ console.error('Error saving message or title:', error)
+
+ // Retry critical save operations with backoff
+ try {
+ for (const promise of savePromises) {
+ await retryDatabaseOperation(
+ () => promise,
+ 'save message/title'
+ )
+ }
+ } catch (retryError) {
+ console.error('Failed to save after retries with backoff:', retryError)
+ // Consider implementing alerting mechanism here
+ // For now, we ensure the error is logged for monitoring
+ }
+ })
+
// Send a finish message to complete the stream
writer.write({ type: 'finish' })
return
@@ -252,28 +274,51 @@ export async function createChatStreamResponse(
await relatedQuestionsPromise
// Save the complete message after both agents finish
+ const savePromises: Promise<any>[] = []
+
if (validResearchMessage && relatedQuestionsMessage) {
const mergedMessage = mergeUIMessages(
validResearchMessage,
relatedQuestionsMessage
)
- await saveMessage(chatId, mergedMessage)
+ savePromises.push(saveMessage(chatId, mergedMessage))
} else if (validResearchMessage) {
// Save research message only if related questions failed
- await saveMessage(chatId, validResearchMessage)
+ savePromises.push(saveMessage(chatId, validResearchMessage))
}
// Generate proper title after conversation starts
- if (!chat && message) {
+ if (!initialChat && message) {
const userContent = getTextFromParts(message.parts)
- const title = await generateChatTitle({
- userMessageContent: userContent,
- modelId
- })
- // Update chat title
- const { updateChatTitle } = await import('../db/actions')
- await updateChatTitle(chatId, title)
+ savePromises.push(
+ generateChatTitle({
+ userMessageContent: userContent,
+ modelId
+ }).then(async title => {
+ const { updateChatTitle } = await import('../db/actions')
+ return updateChatTitle(chatId, title)
+ })
+ )
}
+
+ // Execute saves in background with exponential backoff retry
+ Promise.all(savePromises).catch(async error => {
+ console.error('Error saving message or title:', error)
+
+ // Retry critical save operations with backoff
+ try {
+ for (const promise of savePromises) {
+ await retryDatabaseOperation(
+ () => promise,
+ 'save message/title'
+ )
+ }
+ } catch (retryError) {
+ console.error('Failed to save after retries with backoff:', retryError)
+ // Consider implementing alerting mechanism here
+ // For now, we ensure the error is logged for monitoring
+ }
+ })
} catch (error) {
console.error('Error generating related questions:', error)
// Save research message even if related questions fail
diff --git a/lib/utils/retry.ts b/lib/utils/retry.ts
new file mode 100644
index 00000000..5d8e0801
--- /dev/null
+++ b/lib/utils/retry.ts
@@ -0,0 +1,66 @@
+// Exponential backoff retry utility
+
+export interface RetryOptions {
+ maxRetries?: number
+ initialDelayMs?: number
+ maxDelayMs?: number
+ backoffMultiplier?: number
+ onRetry?: (error: any, attempt: number) => void
+}
+
+export async function retryWithBackoff<T>(
+ fn: () => Promise<T>,
+ options: RetryOptions = {}
+): Promise<T> {
+ const {
+ maxRetries = 3,
+ initialDelayMs = 100,
+ maxDelayMs = 5000,
+ backoffMultiplier = 2,
+ onRetry
+ } = options
+
+ let lastError: any
+
+ for (let attempt = 0; attempt <= maxRetries; attempt++) {
+ try {
+ return await fn()
+ } catch (error) {
+ lastError = error
+
+ if (attempt === maxRetries) {
+ throw error
+ }
+
+ // Calculate delay with exponential backoff
+ const delay = Math.min(
+ initialDelayMs * Math.pow(backoffMultiplier, attempt),
+ maxDelayMs
+ )
+
+ if (onRetry) {
+ onRetry(error, attempt + 1)
+ }
+
+ // Wait before retrying
+ await new Promise(resolve => setTimeout(resolve, delay))
+ }
+ }
+
+ throw lastError
+}
+
+// Specialized retry for database operations
+export async function retryDatabaseOperation<T>(
+ operation: () => Promise<T>,
+ operationName: string
+): Promise<T> {
+ return retryWithBackoff(operation, {
+ maxRetries: 2,
+ initialDelayMs: 200,
+ maxDelayMs: 2000,
+ onRetry: (error, attempt) => {
+ console.log(`Retrying ${operationName} (attempt ${attempt}):`, error.message)
+ }
+ })
+}
\ No newline at end of file