37 tok/s from Qwen3-Next-80B (48 GB) on a 32 GB laptop, 24 without drafts vs 2.8 for llama.cpp; reading 62 vs 16 tok/s. Faster than llama.cpp on all 13 models compared in Q4_K_M. LLMs bigger than your RAM, pure C, CPU or GPU.
C avx2ccpu-inferenceggufinference-enginelinuxllmlocal-llmmixture-of-expertsmoenvmequantizationqwen3simdspeculative-decodingvulkan
/* SPDX-License-Identifier: GPL-3.0-or-later
Copyright (C) 2026 Maurizio Cammalleri */
/*
* vision.c - the vision encoder of Qwen3.5 and Qwen3-VL (vision.h), as their
* Hugging Face processor and model compute it:
* - the picture resized (Pillow's bicubic, image_resize.c) to the size
* smart_resize gives: multiples of patch x merge (32), its pixels within
* the model's bounds, the aspect kept; scaled to [0, 1], normalized;
* - cut into 16x16 patches taken 2x2 block by 2x2 block (the order the
* merger joins them in), each patch given as two equal frames;
* - the patch embedding (a 3D convolution: one matrix), plus the learned
* positions of a 48x48 grid resampled bilinearly (corners aligned) to
* the picture's grid;
* - blocks of LayerNorm, attention over every patch with a rotary
* embedding of the patch's row (first quarter of each half of a head)
* and column, LayerNorm and a tanh-GELU MLP, each with its residual;
* - the merger: LayerNorm of each patch, four neighbours joined, a linear
* layer, exact GELU, a linear layer to the language model's width.
* Weights are bfloat16 as the checkpoint has them, everything else float.
*/
#include "vision_impl.h"
#include <math.h>
#include <unistd.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include "bf16.h"
#include "vmath.h"
#include "common/image.h"
#include "common/pool.h"
#include "jns.h"
const void *janas_vision_tensor(struct janas_vision *v, const char *name,
uint32_t type, uint64_t rows, uint64_t cols)
{
for (uint64_t i = 0; i < v->j.h.n_tensor; i++) {
const struct janas_jns_tensor *t = &v->j.tensors[i];
if (strncmp(t->name, name, sizeof t->name))
continue;
uint64_t n = t->n_dims == 2 ? t->dims[0] * t->dims[1] : t->dims[0];
uint64_t esz = type == 30 ? 2 : 4;
if (t->type != type || n != rows * cols || t->bytes != n * esz ||
(t->n_dims == 2 && t->dims[0] != cols))
return NULL;
return v->data + (t->offset - v->j.h.resident_offset);
}
return NULL;
}
void janas_vision_close(struct janas_vision *v)
{
if (!v)
return;
for (uint32_t l = 0; l < MAX_BLOCKS; l++) {
free(v->l[l].t_qkv);
free(v->l[l].t_out);
free(v->l[l].t_up);
free(v->l[l].t_down);
}
free(v->t_patch);
free(v->t_fc1);
free(v->t_fc2);
janas_vg_close(v);
free(v->data);
janas_jns_close(&v->j);
free(v);
}
/* the resident tensors read into memory: NULL, or why not */
static const char *read_data(struct janas_vision *v)
{
struct janas_jns *j = &v->j;
v->data = malloc(j->h.resident_bytes ? j->h.resident_bytes : 1);
if (!v->data)
return "out of memory";
size_t got = 0;
while (got < j->h.resident_bytes) {
ssize_t k = pread(j->fd, v->data + got, j->h.resident_bytes - got,
(off_t)(j->h.resident_offset + got));
if (k <= 0)
break;
got += (size_t)k;
}
return got == j->h.resident_bytes ? NULL
: "cannot read the encoder's weights";
}
struct janas_vision *janas_vision_open(const char *path, char *err,
size_t err_len)
{
struct janas_vision *v = calloc(1, sizeof(*v));
if (!v) {
snprintf(err, err_len, "out of memory");
return NULL;
}
if (janas_jns_open(path, &v->j, err, err_len) != 0) {
free(v);
return NULL;
}
const char *why = NULL;
v->kind = !strncmp(v->j.h.arch, "qwen3vl_vision", sizeof v->j.h.arch)
? VISION_QWEN
: !strncmp(v->j.h.arch, "gemma4_vision", sizeof v->j.h.arch)
? VISION_GEMMA
: 0;
if (!v->kind)
why = "not a vision encoder this runs";
struct janas_jns *j = &v->j;
if (!why && v->kind == VISION_GEMMA) { /* vision_gemma.c */
why = read_data(v);
if (!why)
why = janas_vg_open(v);
goto opened;
}
if (!why &&
(janas_jns_meta_u32(j, "vision.block_count", &v->n_block) ||
janas_jns_meta_u32(j, "vision.embedding_length", &v->dim) ||
janas_jns_meta_u32(j, "vision.feed_forward_length", &v->ff) ||
janas_jns_meta_u32(j, "vision.attention.head_count", &v->heads) ||
janas_jns_meta_u32(j, "vision.patch_size", &v->patch) ||
janas_jns_meta_u32(j, "vision.temporal_patch_size", &v->tpatch) ||
janas_jns_meta_u32(j, "vision.spatial_merge_size", &v->merge) ||
janas_jns_meta_u32(j, "vision.position_count", &v->npos) ||
janas_jns_meta_u32(j, "vision.projection_dim", &v->proj) ||
janas_jns_meta_u32(j, "vision.min_pixels", &v->min_pixels) ||
janas_jns_meta_u32(j, "vision.max_pixels", &v->max_pixels) ||
janas_jns_meta_f32(j, "vision.attention.layer_norm_epsilon",
&v->eps) ||
janas_jns_meta_f32(j, "vision.rope.freq_base", &v->theta)))
why = "the encoder's sizes are missing";
uint32_t ti = 0, ts = 0, te = 0;
if (!why && (janas_jns_meta_u32(j, "vision.token.image", &ti) ||
janas_jns_meta_u32(j, "vision.token.start", &ts) ||
janas_jns_meta_u32(j, "vision.token.end", &te)))
why = "the image tokens are missing";
v->tok_image = (int32_t)ti;
v->tok_start = (int32_t)ts;
v->tok_end = (int32_t)te;
/* the processor's mean and deviation, per channel */
for (int k = 0; !why && k < 3; k++) {
char key[40];
snprintf(key, sizeof key, "vision.image_mean.%d", k);
if (janas_jns_meta_f32(j, key, &v->mean[k]))
why = "the processor's mean is missing";
snprintf(key, sizeof key, "vision.image_std.%d", k);
if (janas_jns_meta_f32(j, key, &v->stdv[k]) || v->stdv[k] == 0)
why = "the processor's deviation is missing";
}
uint32_t side = 0;
while ((side + 1) * (side + 1) <= v->npos)
side++;
if (!why && (v->n_block < 1 || v->n_block > MAX_BLOCKS || !v->heads ||
v->dim % v->heads || (v->dim / v->heads) % 4 ||
v->dim / v->heads > 512 || v->merge != 2 ||
side * side != v->npos || v->patch < 2))
why = "the encoder's sizes are out of range";
if (!why)
why = read_data(v);
uint64_t pd = 3ull * v->tpatch * v->patch * v->patch;
uint64_t md = (uint64_t)v->dim * v->merge * v->merge;
if (!why) {
v->patch_w = janas_vision_tensor(v, "v.patch_embd", 30, v->dim, pd);
v->patch_b = janas_vision_tensor(v, "v.patch_embd.bias", 0, 1, v->dim);
v->pos = janas_vision_tensor(v, "v.pos_embd", 30, v->npos, v->dim);
v->m_ln_w = janas_vision_tensor(v, "v.merger.ln.weight", 0, 1, v->dim);
v->m_ln_b = janas_vision_tensor(v, "v.merger.ln.bias", 0, 1, v->dim);
v->m_fc1 = janas_vision_tensor(v, "v.merger.fc1", 30, md, md);
v->m_fc1_b = janas_vision_tensor(v, "v.merger.fc1.bias", 0, 1, md);
v->m_fc2 = janas_vision_tensor(v, "v.merger.fc2", 30, v->proj, md);
v->m_fc2_b = janas_vision_tensor(v, "v.merger.fc2.bias", 0, 1, v->proj);
int ok = v->patch_w && v->patch_b && v->pos && v->m_ln_w && v->m_ln_b &&
v->m_fc1 && v->m_fc1_b && v->m_fc2 && v->m_fc2_b;
for (uint32_t l = 0; ok && l < v->n_block; l++) {
struct vlayer *L = &v->l[l];
char n[64];
#define T(field, part, type, r, c) \
snprintf(n, sizeof n, "v.blk.%u." part, l); \
L->field = janas_vision_tensor(v, n, type, r, c); \
ok = ok && L->field
T(ln1_w, "ln1.weight", 0, 1, v->dim);
T(ln1_b, "ln1.bias", 0, 1, v->dim);
T(ln2_w, "ln2.weight", 0, 1, v->dim);
T(ln2_b, "ln2.bias", 0, 1, v->dim);
T(qkv, "attn_qkv", 30, 3 * v->dim, v->dim);
T(qkv_b, "attn_qkv.bias", 0, 1, 3 * v->dim);
T(out, "attn_out", 30, v->dim, v->dim);
T(out_b, "attn_out.bias", 0, 1, v->dim);
T(up, "ffn_up", 30, v->ff, v->dim);
T(up_b, "ffn_up.bias", 0, 1, v->ff);
T(down, "ffn_down", 30, v->dim, v->ff);
T(down_b, "ffn_down.bias", 0, 1, v->dim);
#undef T
}
if (!ok)
why = "a tensor of the encoder is missing or of another shape";
}
if (!why) { /* the matrices transposed once, for the products */
v->t_patch = vk_transposed(v->patch_w, v->dim, pd);
v->t_fc1 = vk_transposed(v->m_fc1, md, md);
v->t_fc2 = vk_transposed(v->m_fc2, v->proj, md);
int ok = v->t_patch && v->t_fc1 && v->t_fc2;
for (uint32_t l = 0; ok && l < v->n_block; l++) {
struct vlayer *L = &v->l[l];
L->t_qkv = vk_transposed(L->qkv, 3 * v->dim, v->dim);
L->t_out = vk_transposed(L->out, v->dim, v->dim);
L->t_up = vk_transposed(L->up, v->ff, v->dim);
L->t_down = vk_transposed(L->down, v->dim, v->ff);
ok = L->t_qkv && L->t_out && L->t_up && L->t_down;
}
if (!ok)
why = "out of memory";
}
opened:
if (why) {
snprintf(err, err_len, "%s: %s", path, why);
janas_vision_close(v);
return NULL;
}
return v;
}
uint32_t janas_vision_dim(const struct janas_vision *v)
{
return v->proj;
}
int janas_vision_both_ways(const struct janas_vision *v)
{
return v->both_ways;
}
void janas_vision_tokens(const struct janas_vision *v, int32_t *image,
int32_t *start, int32_t *end)
{
*image = v->tok_image;
*start = v->tok_start;
*end = v->tok_end;
}
/* Python's round(): halves to the even neighbour */
static double round_even(double x)
{
double r = floor(x + 0.5);
if (r - x == 0.5 && fmod(r, 2) != 0)
r -= 1;
return r;
}
void janas_vision_size(const struct janas_vision *v, int w, int h,
int max_tokens, int *rw, int *rh, int *nx, int *ny)
{
if (v->kind == VISION_GEMMA) {
janas_vg_size(v, w, h, max_tokens, rw, rh, nx, ny);
return;
}
double f = (double)v->patch * v->merge;
double maxp = v->max_pixels, minp = v->min_pixels;
if (max_tokens > 0 && maxp > (double)max_tokens * f * f)
maxp = (double)max_tokens * f * f;
double hb = round_even(h / f) * f, wb = round_even(w / f) * f;
if (hb * wb > maxp) {
double beta = sqrt((double)h * w / maxp);
hb = fmax(f, floor(h / beta / f) * f);
wb = fmax(f, floor(w / beta / f) * f);
} else if (hb * wb < minp) {
double beta = sqrt(minp / ((double)h * w));
hb = ceil(h * beta / f) * f;
wb = ceil(w * beta / f) * f;
}
if (hb < f)
hb = f;
if (wb < f)
wb = f;
*rw = (int)wb;
*rh = (int)hb;
*nx = (int)(wb / f);
*ny = (int)(hb / f);
}
/* ---- the computation ---- */
int janas_vision_encode(struct janas_vision *v, const struct janas_image *im,
int max_tokens, struct janas_pool *pool, float **out,
int *nx, int *ny, char *err, size_t err_len)
{
if (v->kind == VISION_GEMMA)
return janas_vg_encode(v, im, max_tokens, pool, out, nx, ny, err,
err_len);
*out = NULL;
int rw, rh;
janas_vision_size(v, im->w, im->h, max_tokens, &rw, &rh, nx, ny);
struct janas_image rs;
const struct janas_image *src = im;
if (rw != im->w || rh != im->h) {
if (janas_image_resize(im, rw, rh, &rs) != 0) {
snprintf(err, err_len, "out of memory");
return -1;
}
src = &rs;
}
uint32_t P = v->patch, M = v->merge, D = v->dim;
uint32_t gw = (uint32_t)rw / P, gh = (uint32_t)rh / P;
uint32_t n = gw * gh, pd = 3 * v->tpatch * P * P;
uint32_t hd = D / v->heads;
float *pix = malloc(sizeof(float) * (size_t)n * pd);
float *x = malloc(sizeof(float) * (size_t)n * D);
float *t = malloc(sizeof(float) * (size_t)n * D);
float *qkv = malloc(sizeof(float) * (size_t)n * 3 * D);
float *ffb =
malloc(sizeof(float) * (size_t)n * (v->ff > 3 * D ? v->ff : 3 * D));
int *prow = malloc(sizeof(int) * n), *pcol = malloc(sizeof(int) * n);
float *res = NULL, *cs = NULL, *kt = NULL;
struct janas_pool *own = NULL; /* one thread, when none is given */
int rc = -1;
if (!pool && !(pool = own = janas_pool_create(1, NULL))) {
snprintf(err, err_len, "out of memory");
return -1;
}
kt = malloc(sizeof(float) * (size_t)n * D * 2); /* keys, then values */
if (!pix || !x || !t || !qkv || !ffb || !prow || !pcol || !kt) {
snprintf(err, err_len, "out of memory");
goto done;
}
/* patches, 2x2 block by block; each patch [channel][frame][y][x] */
uint32_t idx = 0;
for (uint32_t by = 0; by < gh / M; by++)
for (uint32_t bx = 0; bx < gw / M; bx++)
for (uint32_t my = 0; my < M; my++)
for (uint32_t mx = 0; mx < M; mx++, idx++) {
uint32_t r = by * M + my, c = bx * M + mx;
prow[idx] = (int)r;
pcol[idx] = (int)c;
float *p = pix + (size_t)idx * pd;
for (uint32_t ch = 0; ch < 3; ch++)
for (uint32_t f = 0; f < v->tpatch; f++)
for (uint32_t yy = 0; yy < P; yy++)
for (uint32_t xx = 0; xx < P; xx++) {
uint8_t s = src->rgb[((size_t)(r * P + yy) *
(size_t)rw +
(c * P + xx)) *
3 +
ch];
*p++ = (s / 255.0f - v->mean[ch]) /
v->stdv[ch];
}
}
vk_matmul(pool, v->t_patch, v->patch_b, pix, x, D, pd, n, 0);
/* the learned positions, resampled to the grid (corners aligned) */
uint32_t side = 0;
while ((side + 1) * (side + 1) <= v->npos)
side++;
for (uint32_t i = 0; i < n; i++) {
double sr = gh > 1 ? (double)prow[i] * (side - 1) / (gh - 1) : 0;
double sc = gw > 1 ? (double)pcol[i] * (side - 1) / (gw - 1) : 0;
int r0 = (int)floor(sr), c0 = (int)floor(sc);
int r1 = r0 + 1 < (int)side ? r0 + 1 : (int)side - 1;
int c1 = c0 + 1 < (int)side ? c0 + 1 : (int)side - 1;
float wr = (float)(sr - r0), wc = (float)(sc - c0);
float w00 = (1 - wr) * (1 - wc), w01 = (1 - wr) * wc;
float w10 = wr * (1 - wc), w11 = wr * wc;
const uint16_t *e00 = v->pos + ((size_t)r0 * side + (size_t)c0) * D;
const uint16_t *e01 = v->pos + ((size_t)r0 * side + (size_t)c1) * D;
const uint16_t *e10 = v->pos + ((size_t)r1 * side + (size_t)c0) * D;
const uint16_t *e11 = v->pos + ((size_t)r1 * side + (size_t)c1) * D;
float *xi = x + (size_t)i * D;
for (uint32_t k = 0; k < D; k++)
xi[k] += w00 * vk_bf(e00[k]) + w01 * vk_bf(e01[k]) +
w10 * vk_bf(e10[k]) + w11 * vk_bf(e11[k]);
}
/* the rotary angles: a quarter of a head for the row, a quarter for the
column, the half repeated */
uint32_t q4 = hd / 4;
cs = malloc(sizeof(float) * (size_t)n * hd * 2);
if (!cs) {
snprintf(err, err_len, "out of memory");
goto done;
}
for (uint32_t i = 0; i < n; i++) {
float *c = cs + (size_t)i * hd * 2, *s = c + hd;
for (uint32_t k = 0; k < q4; k++) {
float inv = 1.0f / powf(v->theta, (float)(2 * k) / (float)(hd / 2));
float ar = (float)prow[i] * inv, ac = (float)pcol[i] * inv;
c[k] = c[k + hd / 2] = cosf(ar);
s[k] = s[k + hd / 2] = sinf(ar);
c[q4 + k] = c[q4 + k + hd / 2] = cosf(ac);
s[q4 + k] = s[q4 + k + hd / 2] = sinf(ac);
}
}
for (uint32_t l = 0; l < v->n_block; l++) {
const struct vlayer *L = &v->l[l];
for (uint32_t i = 0; i < n; i++)
vk_layer_norm(t + (size_t)i * D, x + (size_t)i * D, L->ln1_w,
L->ln1_b, D, v->eps);
vk_matmul(pool, L->t_qkv, L->qkv_b, t, qkv, 3 * D, D, n, 0);
/* rotate q and k: x cos + rotate_half(x) sin, per head */
for (uint32_t i = 0; i < n; i++) {
const float *c = cs + (size_t)i * hd * 2, *s = c + hd;
for (uint32_t part = 0; part < 2; part++)
for (uint32_t h = 0; h < v->heads; h++) {
float *q = qkv + (size_t)i * 3 * D + part * D + h * hd;
float tmp[512];
for (uint32_t k = 0; k < hd; k++) {
float rot = k < hd / 2 ? -q[k + hd / 2] : q[k - hd / 2];
tmp[k] = q[k] * c[k] + rot * s[k];
}
memcpy(q, tmp, sizeof(float) * hd);
}
}
vk_attention(pool, qkv, kt, t, n, D, v->heads, 1.0f / sqrtf((float)hd));
vk_matmul(pool, L->t_out, L->out_b, t, ffb, D, D, n, 0);
for (size_t k = 0; k < (size_t)n * D; k++)
x[k] += ffb[k];
for (uint32_t i = 0; i < n; i++)
vk_layer_norm(t + (size_t)i * D, x + (size_t)i * D, L->ln2_w,
L->ln2_b, D, v->eps);
vk_matmul(pool, L->t_up, L->up_b, t, ffb, v->ff, D, n, 0);
for (size_t k = 0; k < (size_t)n * v->ff; k++)
ffb[k] = vk_gelu_tanh(ffb[k]);
vk_matmul(pool, L->t_down, L->down_b, ffb, t, D, v->ff, n, 0);
for (size_t k = 0; k < (size_t)n * D; k++)
x[k] += t[k];
}
/* the merger: each patch normalized, four consecutive ones joined */
uint32_t nm = n / (M * M), md = D * M * M;
for (uint32_t i = 0; i < n; i++)
vk_layer_norm(t + (size_t)i * D, x + (size_t)i * D, v->m_ln_w,
v->m_ln_b, D, v->eps);
float *h1 = malloc(sizeof(float) * (size_t)nm * md);
res = malloc(sizeof(float) * (size_t)nm * v->proj);
if (!h1 || !res) {
free(h1);
snprintf(err, err_len, "out of memory");
goto done;
}
vk_matmul(pool, v->t_fc1, v->m_fc1_b, t, h1, md, md, nm, 0);
for (size_t k = 0; k < (size_t)nm * md; k++)
h1[k] = vk_gelu_erf(h1[k]);
vk_matmul(pool, v->t_fc2, v->m_fc2_b, h1, res, v->proj, md, nm, 0);
free(h1);
*out = res;
res = NULL;
rc = 0;
done:
free(res);
free(cs);
free(kt);
if (own)
janas_pool_destroy(own);
free(pix);
free(x);
free(t);
free(qkv);
free(ffb);
free(prow);
free(pcol);
if (src == &rs)
janas_image_free(&rs);
return rc;
}