Skip to content

An AI-powered search engine with a generative UI

12.1 KB
Raw Blame
import { ModelMessage } from 'ai'
import { getEncoding, type TiktokenEncoding } from 'js-tiktoken'

import modelMetadata from '@/lib/config/model-metadata.json'

import { Model } from '../types/models'

import { estimateAttachmentTokens } from './attachment-tokens'

type AttachmentTokenEstimates = ReadonlyMap<string, number>

interface ModelContextInfo {
  contextWindow: number
  inputTokens?: number
  outputTokens: number
}

type SnapshotProviderId = keyof typeof modelMetadata
type SnapshotModelInfo = { context: number; input?: number; output: number }

const PROVIDER_METADATA_BY_ID: Record<string, SnapshotProviderId> = {
  anthropic: 'anthropic',
  google: 'google',
  openai: 'openai'
}

// Default values for unknown models
const DEFAULT_CONTEXT_WINDOW = 16384
const DEFAULT_OUTPUT_TOKENS = 4096

// Safety buffer percentage (reserved for system prompts and formatting)
const SAFETY_BUFFER_RATIO = 0.1

const MAX_OUTPUT_RESERVE_RATIO = 0.5

// Cache for tiktoken encoders
const encoderCache = new Map<string, any>()

// Mapping of our model IDs to tiktoken encoding names
// js-tiktoken supports 'o200k_base', 'cl100k_base', 'p50k_base', 'r50k_base'
const MODEL_TO_ENCODING: Record<string, TiktokenEncoding> = {
  'gpt-4.1': 'cl100k_base',
  'gpt-4.1-mini': 'cl100k_base',
  'gpt-4.1-nano': 'cl100k_base',
  'gpt-4o-mini': 'cl100k_base',
  'gpt-5.6-luna': 'o200k_base',
  'gpt-6-luna': 'o200k_base',
  'claude-opus-4': 'cl100k_base', // Use GPT-4 tokenizer as approximation for Claude
  'claude-sonnet-4': 'cl100k_base',
  'claude-3-7-sonnet': 'cl100k_base',
  'claude-3-7-sonnet-20250219': 'cl100k_base',
  'claude-3-5-haiku-20241022': 'cl100k_base',
  'gemini-3-flash-preview': 'cl100k_base', // Use GPT-4 tokenizer as approximation for Gemini
  'gemini-3.1-flash-lite': 'cl100k_base',
  'gemini-2.5-flash': 'cl100k_base',
  'gemini-2.5-pro': 'cl100k_base',
  'grok-4-0709': 'cl100k_base', // Use GPT-4 tokenizer as approximation for Grok
  'grok-3': 'cl100k_base',
  'grok-3-mini': 'cl100k_base'
}

/**
 * Get model-specific context window information
 */
function getSnapshotModel(
  providerId: SnapshotProviderId,
  modelId: string
): SnapshotModelInfo | undefined {
  const provider = modelMetadata[providerId] as Record<
    string,
    SnapshotModelInfo
  >
  return Object.prototype.hasOwnProperty.call(provider, modelId)
    ? provider[modelId]
    : undefined
}

function findSnapshotModel(model: Model): SnapshotModelInfo | undefined {
  if (model.providerId === 'gateway') {
    const gatewayModel = getSnapshotModel('vercel', model.id)
    if (gatewayModel) return gatewayModel

    const separatorIndex = model.id.indexOf('/')
    if (separatorIndex !== -1) {
      const providerId = model.id.slice(0, separatorIndex)
      const modelId = model.id.slice(separatorIndex + 1)
      if (Object.prototype.hasOwnProperty.call(modelMetadata, providerId)) {
        return getSnapshotModel(providerId as SnapshotProviderId, modelId)
      }
    }

    return undefined
  }

  const providerId = PROVIDER_METADATA_BY_ID[model.providerId]
  if (providerId) {
    return (
      getSnapshotModel(providerId, model.id) ??
      getSnapshotModel('vercel', `${providerId}/${model.id}`)
    )
  }

  for (const snapshotProviderId of Object.keys(
    modelMetadata
  ) as SnapshotProviderId[]) {
    const metadata = getSnapshotModel(snapshotProviderId, model.id)
    if (metadata) return metadata
  }

  return undefined
}

function getModelContextInfo(model: Model): ModelContextInfo {
  const metadata = findSnapshotModel(model)
  if (!metadata) {
    return {
      contextWindow: DEFAULT_CONTEXT_WINDOW,
      outputTokens: DEFAULT_OUTPUT_TOKENS
    }
  }

  return {
    contextWindow: metadata.context,
    inputTokens: metadata.input,
    outputTokens: metadata.output
  }
}

/**
 * Calculate the maximum allowed tokens for input
 */
export function getMaxAllowedTokens(model: Model): number {
  const { contextWindow, inputTokens, outputTokens } =
    getModelContextInfo(model)

  // Calculate available tokens for input
  const reservedOutputTokens = Math.min(
    outputTokens,
    Math.floor(contextWindow * MAX_OUTPUT_RESERVE_RATIO)
  )
  let availableTokens = Math.min(
    inputTokens ?? Infinity,
    contextWindow - reservedOutputTokens
  )

  // Apply safety buffer
  const safetyBuffer = Math.floor(contextWindow * SAFETY_BUFFER_RATIO)
  availableTokens -= safetyBuffer

  // Ensure minimum viable token count
  return Math.max(availableTokens, 1000)
}

/**
 * Extract text content from various message content types
 */
function extractTextContent(content: ModelMessage['content']): string {
  if (!content) return ''

  // Handle string content
  if (typeof content === 'string') {
    return content
  }

  // Handle array of parts
  if (Array.isArray(content)) {
    return content
      .map(part => {
        if ('text' in part) {
          return part.text
        }
        return ''
      })
      .join(' ')
  }

  // Handle other content types
  return ''
}

/**
 * Get or create encoder for a model
 */
function getEncoder(modelId: string) {
  try {
    const encodingName: TiktokenEncoding =
      MODEL_TO_ENCODING[modelId] || 'cl100k_base'

    if (!encoderCache.has(encodingName)) {
      const encoder = getEncoding(encodingName)
      encoderCache.set(encodingName, encoder)
    }

    return encoderCache.get(encodingName)
  } catch (error) {
    if (process.env.NODE_ENV === 'development') {
      console.warn(
        `Failed to load tokenizer for model ${modelId}, falling back to estimation`,
        error
      )
    }
    return null
  }
}

/**
 * Token count for plain text, using the model's tokenizer when available
 */
export function countTextTokens(text: string, modelId?: string): number {
  if (!text) return 0

  const encoder = modelId ? getEncoder(modelId) : null
  if (encoder) {
    try {
      return encoder.encode(text).length
    } catch {
      // Fall through to the character approximation
    }
  }

  return Math.ceil(text.length / 4)
}

/**
 * Estimate token count for a message
 * Uses tiktoken for accurate counting when available
 */
function estimateTokenCount(
  content: ModelMessage['content'],
  modelId?: string,
  attachmentTokenEstimates?: AttachmentTokenEstimates
): number {
  const text = extractTextContent(content)
  const attachmentTokens = Array.isArray(content)
    ? content.reduce((total, part) => {
        if (part.type !== 'file') return total

        const url = getFileDataUrl(part.data)
        const resolvedEstimate = url
          ? attachmentTokenEstimates?.get(url)
          : undefined

        return (
          total +
          (resolvedEstimate ??
            estimateAttachmentTokens({
              mediaType: part.mediaType,
              size: getInlineFileByteLength(part.data)
            }))
        )
      }, 0)
    : 0

  if (!text) return attachmentTokens

  // Try to use tiktoken for accurate counting
  if (modelId) {
    const encoder = getEncoder(modelId)
    if (encoder) {
      try {
        const tokens = encoder.encode(text)
        const tokenCount = tokens.length
        const overhead = 4 // Message formatting tokens
        return tokenCount + overhead + attachmentTokens
      } catch (error) {
        if (process.env.NODE_ENV === 'development') {
          console.warn(
            'Failed to encode text with tiktoken, falling back to estimation',
            error
          )
        }
      }
    }
  }

  // Fallback: Rough approximation
  // ~4 characters per token for English, adjust for other languages
  const baseCount = Math.ceil(text.length / 4)
  const overhead = 4 // Message formatting tokens

  return baseCount + overhead + attachmentTokens
}

function getFileDataUrl(data: unknown): string | undefined {
  if (typeof data === 'string') return data
  if (data instanceof URL) return data.href
  if (!data || typeof data !== 'object') return undefined

  const fileData = data as { type?: unknown; url?: unknown }
  if (fileData.type !== 'url') return undefined

  if (fileData.url instanceof URL) return fileData.url.href
  return typeof fileData.url === 'string' ? fileData.url : undefined
}

function getInlineFileByteLength(data: unknown): number | undefined {
  if (data instanceof Uint8Array || data instanceof ArrayBuffer) {
    return data.byteLength
  }
  if (!data || typeof data !== 'object') return undefined

  const fileData = data as { type?: unknown; data?: unknown }
  if (fileData.type !== 'data') return undefined

  if (
    fileData.data instanceof Uint8Array ||
    fileData.data instanceof ArrayBuffer
  ) {
    return fileData.data.byteLength
  }

  return undefined
}

/**
 * Smart message truncation with priority for context preservation
 */
export function truncateMessages(
  messages: ModelMessage[],
  maxTokens: number,
  modelId?: string,
  attachmentTokenEstimates?: AttachmentTokenEstimates
): ModelMessage[] {
  // Input validation
  if (!messages || messages.length === 0) return []
  if (maxTokens <= 0) {
    console.error('Invalid maxTokens value:', maxTokens)
    return []
  }

  // Always try to keep the first user message (initial context)
  const firstUserIndex = messages.findIndex(m => m.role === 'user')
  const firstUserMessage = firstUserIndex >= 0 ? messages[firstUserIndex] : null

  // Calculate token counts for all messages
  const messageTokenCounts = messages.map(msg => ({
    message: msg,
    tokens: estimateTokenCount(msg.content, modelId, attachmentTokenEstimates)
  }))

  // Calculate total tokens
  const totalTokens = messageTokenCounts.reduce(
    (sum, item) => sum + item.tokens,
    0
  )

  // If under limit, return all messages
  if (totalTokens <= maxTokens) {
    return messages
  }

  // Strategy: Keep first user message + as many recent messages as possible
  const result: ModelMessage[] = []
  let usedTokens = 0

  // Reserve space for first user message if it exists
  if (firstUserMessage) {
    const firstUserTokens = estimateTokenCount(
      firstUserMessage.content,
      modelId,
      attachmentTokenEstimates
    )
    if (firstUserTokens < maxTokens * 0.3) {
      // Don't let first message take more than 30%
      result.push(firstUserMessage)
      usedTokens += firstUserTokens
    }
  }

  // Add recent messages from the end
  const recentMessages: ModelMessage[] = []

  for (let i = messages.length - 1; i >= 0; i--) {
    const { message, tokens } = messageTokenCounts[i]

    // Skip if this is the first user message (already added)
    if (firstUserMessage && i === firstUserIndex) continue

    if (usedTokens + tokens <= maxTokens) {
      recentMessages.unshift(message)
      usedTokens += tokens
    } else {
      // Try to at least include the last user message if we haven't
      if (
        message.role === 'user' &&
        recentMessages.length > 0 &&
        !recentMessages.some(recent => recent.role === 'user')
      ) {
        // Remove oldest assistant messages to make room
        while (recentMessages.length > 0 && usedTokens + tokens > maxTokens) {
          const removed = recentMessages.shift()
          if (removed) {
            usedTokens -= estimateTokenCount(
              removed.content,
              modelId,
              attachmentTokenEstimates
            )
          }
        }
        if (usedTokens + tokens <= maxTokens) {
          recentMessages.unshift(message)
          usedTokens += tokens
        }
      }
      break
    }
  }

  // Combine results, ensuring conversation flow
  if (firstUserMessage && result.length > 0) {
    // If we kept the first message, add recent ones
    result.push(...recentMessages)
  } else {
    // Otherwise, just use recent messages
    result.push(...recentMessages)
  }

  // Ensure the result starts with a user message
  while (result.length > 0 && result[0].role !== 'user') {
    result.shift()
  }

  return result
}

/**
 * Check if messages need truncation
 */
export function shouldTruncateMessages(
  messages: ModelMessage[],
  model: Model,
  attachmentTokenEstimates?: AttachmentTokenEstimates
): boolean {
  if (!messages || messages.length === 0) return false

  const maxTokens = getMaxAllowedTokens(model)
  const totalTokens = messages.reduce(
    (sum, msg) =>
      sum + estimateTokenCount(msg.content, model.id, attachmentTokenEstimates),
    0
  )
  return totalTokens > maxTokens
}