Skip to content

Project NOMAD is an offline-first knowledge and education server. Wikipedia, thousands of books, courses, maps, and optional local AI, all running on hardware you own with no internet required.

TypeScript
3.6 KB
Raw Blame
import type { MultipartFile } from '@adonisjs/bodyparser/types'
import type { ChatCompletionMessageParam } from 'openai/resources/chat/completions.js'
import sharp from 'sharp'

export const MAX_CHAT_IMAGES = 4
export const MAX_CHAT_IMAGE_BYTES = 8 * 1024 * 1024
export const MAX_CHAT_IMAGE_PIXELS = 40_000_000
export const MAX_NORMALIZED_IMAGE_BYTES = 4 * 1024 * 1024
export const MAX_NORMALIZED_IMAGE_DIMENSION = 2048

const SUPPORTED_IMAGE_FORMATS = new Set(['jpeg', 'png', 'webp'])

export class ChatImageError extends Error {
  constructor(
    message: string,
    public readonly status: 413 | 415 | 422
  ) {
    super(message)
    this.name = 'ChatImageError'
  }
}

export type NormalizedChatImage = {
  name: string
  dataUrl: string
}

export async function normalizeChatImages(files: MultipartFile[]): Promise<NormalizedChatImage[]> {
  if (files.length > MAX_CHAT_IMAGES) {
    throw new ChatImageError(`Attach no more than ${MAX_CHAT_IMAGES} images.`, 422)
  }

  const normalized: NormalizedChatImage[] = []
  for (const file of files) {
    normalized.push(await normalizeChatImage(file))
  }
  return normalized
}

async function normalizeChatImage(file: MultipartFile): Promise<NormalizedChatImage> {
  if (!file.tmpPath) {
    throw new ChatImageError(`Could not process "${file.clientName}".`, 422)
  }
  if (file.size > MAX_CHAT_IMAGE_BYTES) {
    throw new ChatImageError(`"${file.clientName}" exceeds the 8 MB per-image limit.`, 413)
  }
  if (!file.isValid) {
    throw new ChatImageError(`"${file.clientName}" is not a valid image upload.`, 422)
  }

  try {
    const pipeline = sharp(file.tmpPath, {
      animated: false,
      failOn: 'warning',
      limitInputPixels: MAX_CHAT_IMAGE_PIXELS,
    })
    const metadata = await pipeline.metadata()

    if (!metadata.format || !SUPPORTED_IMAGE_FORMATS.has(metadata.format)) {
      throw new ChatImageError(
        `"${file.clientName}" is not supported. Use JPEG, PNG, or WebP.`,
        415
      )
    }
    if ((metadata.pages ?? 1) > 1) {
      throw new ChatImageError(`Animated images are not supported.`, 415)
    }

    const normalized = await pipeline
      .rotate()
      .resize({
        width: MAX_NORMALIZED_IMAGE_DIMENSION,
        height: MAX_NORMALIZED_IMAGE_DIMENSION,
        fit: 'inside',
        withoutEnlargement: true,
      })
      .flatten({ background: '#ffffff' })
      .jpeg({ quality: 85 })
      .toBuffer()

    if (normalized.byteLength > MAX_NORMALIZED_IMAGE_BYTES) {
      throw new ChatImageError(
        `"${file.clientName}" remains too large after image processing.`,
        413
      )
    }

    return {
      name: file.clientName,
      dataUrl: `data:image/jpeg;base64,${normalized.toString('base64')}`,
    }
  } catch (error) {
    if (error instanceof ChatImageError) throw error
    throw new ChatImageError(`"${file.clientName}" could not be decoded as an image.`, 422)
  }
}

export function attachImagesToLatestUserMessage(
  messages: Array<{ role: 'system' | 'user' | 'assistant'; content: string }>,
  images: NormalizedChatImage[]
): ChatCompletionMessageParam[] {
  if (images.length === 0) return messages

  let latestUserIndex = -1
  messages.forEach((message, index) => {
    if (message.role === 'user') latestUserIndex = index
  })
  if (latestUserIndex < 0) {
    throw new ChatImageError('Images require a user message.', 422)
  }

  return messages.map((message, index) => {
    if (index !== latestUserIndex) return message
    return {
      role: 'user',
      content: [
        { type: 'text', text: message.content },
        ...images.map((image) => ({
          type: 'image_url' as const,
          image_url: { url: image.dataUrl },
        })),
      ],
    }
  })
}