An AI-powered search engine with a generative UI
TypeScript generative-aigenerative-uinextjsollamareactsearxngshadcn-uitailwindcsstavilytypescriptupstashvercel-ai-sdk
import { ModelMessage } from 'ai'
import { getEncoding, type TiktokenEncoding } from 'js-tiktoken'
import modelMetadata from '@/lib/config/model-metadata.json'
import { Model } from '../types/models'
import { estimateAttachmentTokens } from './attachment-tokens'
type AttachmentTokenEstimates = ReadonlyMap<string, number>
interface ModelContextInfo {
contextWindow: number
inputTokens?: number
outputTokens: number
}
type SnapshotProviderId = keyof typeof modelMetadata
type SnapshotModelInfo = { context: number; input?: number; output: number }
const PROVIDER_METADATA_BY_ID: Record<string, SnapshotProviderId> = {
anthropic: 'anthropic',
google: 'google',
openai: 'openai'
}
// Default values for unknown models
const DEFAULT_CONTEXT_WINDOW = 16384
const DEFAULT_OUTPUT_TOKENS = 4096
// Safety buffer percentage (reserved for system prompts and formatting)
const SAFETY_BUFFER_RATIO = 0.1
const MAX_OUTPUT_RESERVE_RATIO = 0.5
// Cache for tiktoken encoders
const encoderCache = new Map<string, any>()
// Mapping of our model IDs to tiktoken encoding names
// js-tiktoken supports 'o200k_base', 'cl100k_base', 'p50k_base', 'r50k_base'
const MODEL_TO_ENCODING: Record<string, TiktokenEncoding> = {
'gpt-4.1': 'cl100k_base',
'gpt-4.1-mini': 'cl100k_base',
'gpt-4.1-nano': 'cl100k_base',
'gpt-4o-mini': 'cl100k_base',
'gpt-5.6-luna': 'o200k_base',
'gpt-6-luna': 'o200k_base',
'claude-opus-4': 'cl100k_base', // Use GPT-4 tokenizer as approximation for Claude
'claude-sonnet-4': 'cl100k_base',
'claude-3-7-sonnet': 'cl100k_base',
'claude-3-7-sonnet-20250219': 'cl100k_base',
'claude-3-5-haiku-20241022': 'cl100k_base',
'gemini-3-flash-preview': 'cl100k_base', // Use GPT-4 tokenizer as approximation for Gemini
'gemini-3.1-flash-lite': 'cl100k_base',
'gemini-2.5-flash': 'cl100k_base',
'gemini-2.5-pro': 'cl100k_base',
'grok-4-0709': 'cl100k_base', // Use GPT-4 tokenizer as approximation for Grok
'grok-3': 'cl100k_base',
'grok-3-mini': 'cl100k_base'
}
/**
* Get model-specific context window information
*/
function getSnapshotModel(
providerId: SnapshotProviderId,
modelId: string
): SnapshotModelInfo | undefined {
const provider = modelMetadata[providerId] as Record<
string,
SnapshotModelInfo
>
return Object.prototype.hasOwnProperty.call(provider, modelId)
? provider[modelId]
: undefined
}
function findSnapshotModel(model: Model): SnapshotModelInfo | undefined {
if (model.providerId === 'gateway') {
const gatewayModel = getSnapshotModel('vercel', model.id)
if (gatewayModel) return gatewayModel
const separatorIndex = model.id.indexOf('/')
if (separatorIndex !== -1) {
const providerId = model.id.slice(0, separatorIndex)
const modelId = model.id.slice(separatorIndex + 1)
if (Object.prototype.hasOwnProperty.call(modelMetadata, providerId)) {
return getSnapshotModel(providerId as SnapshotProviderId, modelId)
}
}
return undefined
}
const providerId = PROVIDER_METADATA_BY_ID[model.providerId]
if (providerId) {
return (
getSnapshotModel(providerId, model.id) ??
getSnapshotModel('vercel', `${providerId}/${model.id}`)
)
}
for (const snapshotProviderId of Object.keys(
modelMetadata
) as SnapshotProviderId[]) {
const metadata = getSnapshotModel(snapshotProviderId, model.id)
if (metadata) return metadata
}
return undefined
}
function getModelContextInfo(model: Model): ModelContextInfo {
const metadata = findSnapshotModel(model)
if (!metadata) {
return {
contextWindow: DEFAULT_CONTEXT_WINDOW,
outputTokens: DEFAULT_OUTPUT_TOKENS
}
}
return {
contextWindow: metadata.context,
inputTokens: metadata.input,
outputTokens: metadata.output
}
}
/**
* Calculate the maximum allowed tokens for input
*/
export function getMaxAllowedTokens(model: Model): number {
const { contextWindow, inputTokens, outputTokens } =
getModelContextInfo(model)
// Calculate available tokens for input
const reservedOutputTokens = Math.min(
outputTokens,
Math.floor(contextWindow * MAX_OUTPUT_RESERVE_RATIO)
)
let availableTokens = Math.min(
inputTokens ?? Infinity,
contextWindow - reservedOutputTokens
)
// Apply safety buffer
const safetyBuffer = Math.floor(contextWindow * SAFETY_BUFFER_RATIO)
availableTokens -= safetyBuffer
// Ensure minimum viable token count
return Math.max(availableTokens, 1000)
}
/**
* Extract text content from various message content types
*/
function extractTextContent(content: ModelMessage['content']): string {
if (!content) return ''
// Handle string content
if (typeof content === 'string') {
return content
}
// Handle array of parts
if (Array.isArray(content)) {
return content
.map(part => {
if ('text' in part) {
return part.text
}
return ''
})
.join(' ')
}
// Handle other content types
return ''
}
/**
* Get or create encoder for a model
*/
function getEncoder(modelId: string) {
try {
const encodingName: TiktokenEncoding =
MODEL_TO_ENCODING[modelId] || 'cl100k_base'
if (!encoderCache.has(encodingName)) {
const encoder = getEncoding(encodingName)
encoderCache.set(encodingName, encoder)
}
return encoderCache.get(encodingName)
} catch (error) {
if (process.env.NODE_ENV === 'development') {
console.warn(
`Failed to load tokenizer for model ${modelId}, falling back to estimation`,
error
)
}
return null
}
}
/**
* Token count for plain text, using the model's tokenizer when available
*/
export function countTextTokens(text: string, modelId?: string): number {
if (!text) return 0
const encoder = modelId ? getEncoder(modelId) : null
if (encoder) {
try {
return encoder.encode(text).length
} catch {
// Fall through to the character approximation
}
}
return Math.ceil(text.length / 4)
}
/**
* Estimate token count for a message
* Uses tiktoken for accurate counting when available
*/
function estimateTokenCount(
content: ModelMessage['content'],
modelId?: string,
attachmentTokenEstimates?: AttachmentTokenEstimates
): number {
const text = extractTextContent(content)
const attachmentTokens = Array.isArray(content)
? content.reduce((total, part) => {
if (part.type !== 'file') return total
const url = getFileDataUrl(part.data)
const resolvedEstimate = url
? attachmentTokenEstimates?.get(url)
: undefined
return (
total +
(resolvedEstimate ??
estimateAttachmentTokens({
mediaType: part.mediaType,
size: getInlineFileByteLength(part.data)
}))
)
}, 0)
: 0
if (!text) return attachmentTokens
// Try to use tiktoken for accurate counting
if (modelId) {
const encoder = getEncoder(modelId)
if (encoder) {
try {
const tokens = encoder.encode(text)
const tokenCount = tokens.length
const overhead = 4 // Message formatting tokens
return tokenCount + overhead + attachmentTokens
} catch (error) {
if (process.env.NODE_ENV === 'development') {
console.warn(
'Failed to encode text with tiktoken, falling back to estimation',
error
)
}
}
}
}
// Fallback: Rough approximation
// ~4 characters per token for English, adjust for other languages
const baseCount = Math.ceil(text.length / 4)
const overhead = 4 // Message formatting tokens
return baseCount + overhead + attachmentTokens
}
function getFileDataUrl(data: unknown): string | undefined {
if (typeof data === 'string') return data
if (data instanceof URL) return data.href
if (!data || typeof data !== 'object') return undefined
const fileData = data as { type?: unknown; url?: unknown }
if (fileData.type !== 'url') return undefined
if (fileData.url instanceof URL) return fileData.url.href
return typeof fileData.url === 'string' ? fileData.url : undefined
}
function getInlineFileByteLength(data: unknown): number | undefined {
if (data instanceof Uint8Array || data instanceof ArrayBuffer) {
return data.byteLength
}
if (!data || typeof data !== 'object') return undefined
const fileData = data as { type?: unknown; data?: unknown }
if (fileData.type !== 'data') return undefined
if (
fileData.data instanceof Uint8Array ||
fileData.data instanceof ArrayBuffer
) {
return fileData.data.byteLength
}
return undefined
}
/**
* Smart message truncation with priority for context preservation
*/
export function truncateMessages(
messages: ModelMessage[],
maxTokens: number,
modelId?: string,
attachmentTokenEstimates?: AttachmentTokenEstimates
): ModelMessage[] {
// Input validation
if (!messages || messages.length === 0) return []
if (maxTokens <= 0) {
console.error('Invalid maxTokens value:', maxTokens)
return []
}
// Always try to keep the first user message (initial context)
const firstUserIndex = messages.findIndex(m => m.role === 'user')
const firstUserMessage = firstUserIndex >= 0 ? messages[firstUserIndex] : null
// Calculate token counts for all messages
const messageTokenCounts = messages.map(msg => ({
message: msg,
tokens: estimateTokenCount(msg.content, modelId, attachmentTokenEstimates)
}))
// Calculate total tokens
const totalTokens = messageTokenCounts.reduce(
(sum, item) => sum + item.tokens,
0
)
// If under limit, return all messages
if (totalTokens <= maxTokens) {
return messages
}
// Strategy: Keep first user message + as many recent messages as possible
const result: ModelMessage[] = []
let usedTokens = 0
// Reserve space for first user message if it exists
if (firstUserMessage) {
const firstUserTokens = estimateTokenCount(
firstUserMessage.content,
modelId,
attachmentTokenEstimates
)
if (firstUserTokens < maxTokens * 0.3) {
// Don't let first message take more than 30%
result.push(firstUserMessage)
usedTokens += firstUserTokens
}
}
// Add recent messages from the end
const recentMessages: ModelMessage[] = []
for (let i = messages.length - 1; i >= 0; i--) {
const { message, tokens } = messageTokenCounts[i]
// Skip if this is the first user message (already added)
if (firstUserMessage && i === firstUserIndex) continue
if (usedTokens + tokens <= maxTokens) {
recentMessages.unshift(message)
usedTokens += tokens
} else {
// Try to at least include the last user message if we haven't
if (
message.role === 'user' &&
recentMessages.length > 0 &&
!recentMessages.some(recent => recent.role === 'user')
) {
// Remove oldest assistant messages to make room
while (recentMessages.length > 0 && usedTokens + tokens > maxTokens) {
const removed = recentMessages.shift()
if (removed) {
usedTokens -= estimateTokenCount(
removed.content,
modelId,
attachmentTokenEstimates
)
}
}
if (usedTokens + tokens <= maxTokens) {
recentMessages.unshift(message)
usedTokens += tokens
}
}
break
}
}
// Combine results, ensuring conversation flow
if (firstUserMessage && result.length > 0) {
// If we kept the first message, add recent ones
result.push(...recentMessages)
} else {
// Otherwise, just use recent messages
result.push(...recentMessages)
}
// Ensure the result starts with a user message
while (result.length > 0 && result[0].role !== 'user') {
result.shift()
}
return result
}
/**
* Check if messages need truncation
*/
export function shouldTruncateMessages(
messages: ModelMessage[],
model: Model,
attachmentTokenEstimates?: AttachmentTokenEstimates
): boolean {
if (!messages || messages.length === 0) return false
const maxTokens = getMaxAllowedTokens(model)
const totalTokens = messages.reduce(
(sum, msg) =>
sum + estimateTokenCount(msg.content, model.id, attachmentTokenEstimates),
0
)
return totalTokens > maxTokens
}