Skip to content

Project NOMAD is an offline-first knowledge and education server. Wikipedia, thousands of books, courses, maps, and optional local AI, all running on hardware you own with no internet required.

TypeScript
8.9 KB
Raw Blame
import assert from 'node:assert/strict'
import test from 'node:test'
import type { MultipartFile } from '@adonisjs/bodyparser/types'
import sharp from 'sharp'
import { mkdtemp, rm, writeFile } from 'node:fs/promises'
import { tmpdir } from 'node:os'
import { join } from 'node:path'
import {
  attachImagesToLatestUserMessage,
  ChatImageError,
  normalizeChatImages,
} from '../../app/utils/chat_images.js'
import {
  capabilitiesFromOllamaShow,
  installedModelsFromOpenAIResponse,
  resolveModelCapabilities,
  visionCapabilityFromLlamaProps,
} from '../../app/utils/model_capabilities.js'

function uploadedFile(path: string, size: number, name = 'sample.png'): MultipartFile {
  return {
    tmpPath: path,
    size,
    clientName: name,
    isValid: true,
  } as MultipartFile
}

test('normalizes a supported image to a bounded JPEG data URL', async () => {
  const directory = await mkdtemp(join(tmpdir(), 'nomad-chat-image-'))
  const path = join(directory, 'sample.png')
  const input = await sharp({
    create: {
      width: 32,
      height: 24,
      channels: 3,
      background: '#556b2f',
    },
  })
    .png()
    .toBuffer()
  await writeFile(path, input)

  try {
    const [image] = await normalizeChatImages([uploadedFile(path, input.byteLength)])
    assert.equal(image.name, 'sample.png')
    assert.match(image.dataUrl, /^data:image\/jpeg;base64,/)
    const decoded = Buffer.from(image.dataUrl.split(',')[1], 'base64')
    const metadata = await sharp(decoded).metadata()
    assert.equal(metadata.format, 'jpeg')
    assert.equal(metadata.width, 32)
    assert.equal(metadata.height, 24)
  } finally {
    await rm(directory, { recursive: true, force: true })
  }
})

test('normalizes image bytes when the filename extension does not match the encoding', async () => {
  const directory = await mkdtemp(join(tmpdir(), 'nomad-chat-image-'))
  const path = join(directory, 'mislabeled.png')
  const input = await sharp({
    create: {
      width: 32,
      height: 24,
      channels: 3,
      background: '#556b2f',
    },
  })
    .webp()
    .toBuffer()
  await writeFile(path, input)

  try {
    const [image] = await normalizeChatImages([
      uploadedFile(path, input.byteLength, 'mislabeled.png'),
    ])
    assert.equal(image.name, 'mislabeled.png')
    assert.match(image.dataUrl, /^data:image\/jpeg;base64,/)
    const decoded = Buffer.from(image.dataUrl.split(',')[1], 'base64')
    const metadata = await sharp(decoded).metadata()
    assert.equal(metadata.format, 'jpeg')
    assert.equal(metadata.width, 32)
    assert.equal(metadata.height, 24)
  } finally {
    await rm(directory, { recursive: true, force: true })
  }
})

test('rejects payloads with more than four images', async () => {
  const files = Array.from({ length: 5 }, (_, index) =>
    uploadedFile(`/tmp/image-${index}.png`, 10, `image-${index}.png`)
  )
  await assert.rejects(
    () => normalizeChatImages(files),
    (error: unknown) => {
      assert.ok(error instanceof ChatImageError)
      assert.equal(error.status, 422)
      return true
    }
  )
})

test('attaches images only to the latest user message', () => {
  const messages = [
    { role: 'user' as const, content: 'Earlier question' },
    { role: 'assistant' as const, content: 'Earlier answer' },
    { role: 'user' as const, content: 'What is shown?' },
  ]
  const result = attachImagesToLatestUserMessage(messages, [
    { name: 'sample.webp', dataUrl: 'data:image/webp;base64,AAAA' },
  ])

  assert.equal(result[0].content, 'Earlier question')
  assert.equal(result[1].content, 'Earlier answer')
  assert.deepEqual(result[2], {
    role: 'user',
    content: [
      { type: 'text', text: 'What is shown?' },
      {
        type: 'image_url',
        image_url: { url: 'data:image/webp;base64,AAAA' },
      },
    ],
  })
})

test('detects vision capability from Ollama and llama.cpp metadata', () => {
  assert.deepEqual(capabilitiesFromOllamaShow({ capabilities: ['completion', 'vision'] }), {
    thinking: false,
    vision: 'supported',
  })
  assert.deepEqual(capabilitiesFromOllamaShow({ capabilities: ['completion', 'thinking'] }), {
    thinking: true,
    vision: 'unsupported',
  })
  assert.equal(visionCapabilityFromLlamaProps({ modalities: { vision: true } }), 'supported')
  assert.equal(visionCapabilityFromLlamaProps({ modalities: { vision: false } }), 'unsupported')
  assert.equal(visionCapabilityFromLlamaProps({ modalities: ['text', 'image'] }), 'supported')
  assert.equal(visionCapabilityFromLlamaProps({ model: 'unknown' }), null)
})

test('resolves per-model capabilities advertised by a hybrid router', async () => {
  let showProbeCalls = 0
  let propsProbeCalls = 0
  const probes = {
    ollamaShow: async () => {
      showProbeCalls += 1
      return null
    },
    llamaProps: async () => {
      propsProbeCalls += 1
      return null
    },
  }

  const visionModel = await resolveModelCapabilities(
    { capabilities: ['completion', 'vision', 'tools', 'thinking'] },
    probes
  )
  const textModel = await resolveModelCapabilities(
    { capabilities: ['completion', 'tools', 'thinking'] },
    probes
  )

  assert.deepEqual(visionModel, { thinking: true, vision: 'supported' })
  assert.deepEqual(textModel, { thinking: true, vision: 'unsupported' })
  // A listed vision model is trusted without probing. A listed text-only model is
  // probed once, and the router's answer stands when the probes have nothing better.
  assert.equal(showProbeCalls, 1)
  assert.equal(propsProbeCalls, 1)
})

test('probes a model the list reports without vision, as Ollama does for gemma3', async () => {
  // Measured on Ollama 0.33.3: /api/tags lists gemma3:4b as ['completion'] while
  // /api/show for the same model reports ['completion', 'vision'].
  const gemma = await resolveModelCapabilities(
    { capabilities: ['completion'] },
    {
      ollamaShow: async () => ({ capabilities: ['completion', 'vision'] }),
      llamaProps: async () => {
        throw new Error('not used')
      },
    }
  )

  assert.deepEqual(gemma, { thinking: false, vision: 'supported' })
})

test('preserves per-model capabilities from an OpenAI-compatible model list', () => {
  const models = installedModelsFromOpenAIResponse({
    data: [
      {
        id: 'qwen3.6:35b-a3b-hauhau-aggressive',
        capabilities: ['completion', 'vision', 'tools', 'thinking'],
      },
      {
        id: 'qwopus3.6-coder-compat-mtp:27b',
        capabilities: ['completion', 'tools', 'thinking'],
      },
    ],
  })

  assert.deepEqual(models, [
    {
      name: 'qwen3.6:35b-a3b-hauhau-aggressive',
      size: 0,
      capabilities: ['completion', 'vision', 'tools', 'thinking'],
    },
    {
      name: 'qwopus3.6-coder-compat-mtp:27b',
      size: 0,
      capabilities: ['completion', 'tools', 'thinking'],
    },
  ])
})

test('falls back per model from Ollama metadata to llama.cpp properties', async () => {
  let propsProbeCalls = 0
  const ollamaModel = await resolveModelCapabilities(null, {
    ollamaShow: async () => ({ capabilities: ['completion', 'vision'] }),
    llamaProps: async () => {
      propsProbeCalls += 1
      return { modalities: { vision: false } }
    },
  })
  const llamaModel = await resolveModelCapabilities(null, {
    ollamaShow: async () => {
      throw new Error('404')
    },
    llamaProps: async () => {
      propsProbeCalls += 1
      return { modalities: { vision: false } }
    },
  })

  assert.deepEqual(ollamaModel, { thinking: false, vision: 'supported' })
  assert.deepEqual(llamaModel, { thinking: false, vision: 'unsupported' })
  assert.equal(propsProbeCalls, 1)
})

test('classifies the four supported backend capability cases', async () => {
  const backendCases = [
    {
      name: 'Ollama vision',
      advertised: null,
      ollamaShow: async () => ({ capabilities: ['completion', 'vision'] }),
      llamaProps: async () => {
        throw new Error('not used')
      },
      expected: { thinking: false, vision: 'supported' },
    },
    {
      name: 'llama.cpp vision',
      advertised: null,
      ollamaShow: async () => {
        throw new Error('404')
      },
      llamaProps: async () => ({ modalities: { vision: true } }),
      expected: { thinking: false, vision: 'supported' },
    },
    {
      name: 'known text-only',
      advertised: { capabilities: ['completion', 'thinking'] },
      ollamaShow: async () => ({ capabilities: ['completion', 'thinking'] }),
      llamaProps: async () => {
        throw new Error('not used')
      },
      expected: { thinking: true, vision: 'unsupported' },
    },
    {
      name: 'unknown OpenAI-compatible',
      advertised: null,
      ollamaShow: async () => {
        throw new Error('404')
      },
      llamaProps: async () => {
        throw new Error('404')
      },
      expected: null,
    },
  ] as const

  for (const backend of backendCases) {
    const actual = await resolveModelCapabilities(backend.advertised, {
      ollamaShow: backend.ollamaShow,
      llamaProps: backend.llamaProps,
    })
    assert.deepEqual(actual, backend.expected, backend.name)
  }
})