37 tok/s from Qwen3-Next-80B (48 GB) on a 32 GB laptop, 24 without drafts vs 2.8 for llama.cpp; reading 62 vs 16 tok/s. Faster than llama.cpp on all 13 models compared in Q4_K_M. LLMs bigger than your RAM, pure C, CPU or GPU.
C avx2ccpu-inferenceggufinference-enginelinuxllmlocal-llmmixture-of-expertsmoenvmequantizationqwen3simdspeculative-decodingvulkan
/* SPDX-License-Identifier: GPL-3.0-or-later
Copyright (C) 2026 Maurizio Cammalleri */
/*
* vision_gemma.c - Gemma 4's vision encoder (vision_impl.h), as its Hugging
* Face processor and model compute it:
* - the picture resized (Pillow's bicubic) to the largest size, aspect
* kept, whose 16x16 patches are at most the soft tokens times 3 x 3, a
* multiple of 48 a side; pixels scaled to [-1, 1];
* - each patch (row, column, channel) through one matrix, plus the
* learned embeddings of its column and of its row;
* - blocks of RMS norm, attention (queries and keys RMS-normed with a
* scale, values without, a rotary embedding of the column on the first
* half of each head and of the row on the second, no scaling of the
* scores), RMS norm, residual; RMS norm, a gated tanh-GELU MLP, RMS
* norm, residual - every linear clamping its input and its output to
* bounds of its own;
* - the patches averaged 3 x 3, times sqrt(width), standardized where
* the model says, RMS-normed and projected to the language model's
* width.
*/
#include "vision_impl.h"
#include <math.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include "common/image.h"
#include "common/pool.h"
/* the soft tokens a picture may have, as the processor allows them */
static const uint32_t SOFT[] = {70, 140, 280, 560, 1120};
const char *janas_vg_open(struct janas_vision *v)
{
struct janas_jns *j = &v->j;
uint32_t ti = 0, ts = 0, te = 0, stdz = 0, unified = 0;
janas_jns_meta_u32(j, "vision.unified", &unified);
if (unified) /* the 12B's, without blocks */
return janas_vgu_open(v);
if (janas_jns_meta_u32(j, "vision.block_count", &v->n_block) ||
janas_jns_meta_u32(j, "vision.embedding_length", &v->dim) ||
janas_jns_meta_u32(j, "vision.feed_forward_length", &v->ff) ||
janas_jns_meta_u32(j, "vision.attention.head_count", &v->heads) ||
janas_jns_meta_u32(j, "vision.patch_size", &v->patch) ||
janas_jns_meta_u32(j, "vision.pooling_kernel_size", &v->pool_k) ||
janas_jns_meta_u32(j, "vision.position_size", &v->pos_size) ||
janas_jns_meta_u32(j, "vision.max_soft_tokens", &v->max_soft) ||
janas_jns_meta_u32(j, "vision.projection_dim", &v->proj) ||
janas_jns_meta_u32(j, "vision.standardize", &stdz) ||
janas_jns_meta_f32(j, "vision.attention.layer_norm_epsilon", &v->eps) ||
janas_jns_meta_f32(j, "vision.rope.freq_base", &v->theta))
return "the encoder's sizes are missing";
if (janas_jns_meta_u32(j, "vision.token.image", &ti) ||
janas_jns_meta_u32(j, "vision.token.start", &ts) ||
janas_jns_meta_u32(j, "vision.token.end", &te))
return "the image tokens are missing";
uint32_t both = 0; /* absent in the files of before */
janas_jns_meta_u32(j, "vision.bidirectional", &both);
v->both_ways = both != 0;
v->tok_image = (int32_t)ti;
v->tok_start = (int32_t)ts;
v->tok_end = (int32_t)te;
if (v->n_block < 1 || v->n_block > MAX_BLOCKS || !v->heads ||
v->dim % v->heads || (v->dim / v->heads) % 4 ||
v->dim / v->heads > 512 || v->pool_k < 1 || v->pool_k > 8 ||
v->patch < 2 || !v->pos_size || !v->max_soft || !v->proj)
return "the encoder's sizes are out of range";
uint64_t pd = 3ull * v->patch * v->patch;
const uint16_t *pw = janas_vision_tensor(v, "v.patch_embd", 30, v->dim, pd);
v->g_pos =
janas_vision_tensor(v, "v.pos_embd", 30, 2ull * v->pos_size, v->dim);
const uint16_t *proj =
janas_vision_tensor(v, "v.proj", 30, v->proj, v->dim);
if (stdz) {
v->std_bias = janas_vision_tensor(v, "v.std_bias", 0, 1, v->dim);
v->std_scale = janas_vision_tensor(v, "v.std_scale", 0, 1, v->dim);
if (!v->std_bias || !v->std_scale)
return "the standardization is missing";
}
if (!pw || !v->g_pos || !proj)
return "a tensor of the encoder is missing or of another shape";
v->t_g_patch = vk_transposed(pw, v->dim, pd);
v->t_g_proj = vk_transposed(proj, v->proj, v->dim);
if (!v->t_g_patch || !v->t_g_proj)
return "out of memory";
static const char *const LIN[GL_N] = {"q", "k", "v", "o",
"gate", "up", "down"};
uint32_t hd = v->dim / v->heads;
for (uint32_t l = 0; l < v->n_block; l++) {
struct glayer *L = &v->gl[l];
char n[64];
#define NORM(field, part, len) \
snprintf(n, sizeof n, "v.blk.%u." part, l); \
if (!(L->field = janas_vision_tensor(v, n, 0, 1, len))) \
return "a norm of the encoder is missing";
NORM(n_in, "in_norm", v->dim);
NORM(n_post_att, "post_att_norm", v->dim);
NORM(n_pre_ff, "pre_ff_norm", v->dim);
NORM(n_post_ff, "post_ff_norm", v->dim);
NORM(q_norm, "q_norm", hd);
NORM(k_norm, "k_norm", hd);
#undef NORM
snprintf(n, sizeof n, "v.blk.%u.clip", l);
if (!(L->clip = janas_vision_tensor(v, n, 0, 1, GL_N * 4)))
return "the clamps of the encoder are missing";
for (int k = 0; k < GL_N; k++) {
uint64_t rows = k == GL_GATE || k == GL_UP ? v->ff : v->dim;
uint64_t cols = k == GL_DOWN ? v->ff : v->dim;
snprintf(n, sizeof n, "v.blk.%u.%s", l, LIN[k]);
const uint16_t *w = janas_vision_tensor(v, n, 30, rows, cols);
if (!w)
return "a matrix of the encoder is missing";
if (!(L->t[k] = vk_transposed(w, rows, cols)))
return "out of memory";
}
}
return NULL;
}
void janas_vg_close(struct janas_vision *v)
{
for (uint32_t l = 0; l < MAX_BLOCKS; l++)
for (int k = 0; k < GL_N; k++)
free(v->gl[l].t[k]);
free(v->t_g_patch);
free(v->t_g_proj);
free(v->t_u_dense);
}
void janas_vg_size(const struct janas_vision *v, int w, int h, int max_tokens,
int *rw, int *rh, int *nx, int *ny)
{
uint32_t soft = v->max_soft;
if (max_tokens > 0 && (uint32_t)max_tokens < soft) {
soft = SOFT[0];
for (size_t i = 0; i < sizeof SOFT / sizeof SOFT[0]; i++)
if (SOFT[i] <= (uint32_t)max_tokens)
soft = SOFT[i];
}
double k = v->pool_k, P = v->patch, side = k * P;
double max_patches = (double)soft * k * k;
double target = max_patches * P * P;
double f = sqrt(target / ((double)w * h));
int th = (int)(floor(f * h / side) * side);
int tw = (int)(floor(f * w / side) * side);
int longest = (int)((uint32_t)max_patches / (uint32_t)(k * k) * side);
if (th == 0 && tw == 0)
th = tw = (int)side;
else if (th == 0) {
th = (int)side;
tw = (int)(floor((double)w / h) * side);
if (tw > longest)
tw = longest;
} else if (tw == 0) {
tw = (int)side;
th = (int)(floor((double)h / w) * side);
if (th > longest)
th = longest;
}
*rw = tw;
*rh = th;
*nx = (int)(tw / side);
*ny = (int)(th / side);
}
/* x clamped into y */
static void clamp_into(float *y, const float *x, size_t n, float lo, float hi)
{
for (size_t i = 0; i < n; i++)
y[i] = x[i] < lo ? lo : x[i] > hi ? hi : x[i];
}
/* y[n] (ldy apart) = clamp(W clamp(x)) with linear k's bounds; tmp holds
n x cols floats */
static void clipped(struct janas_pool *pool, const struct glayer *L, int k,
const float *x, float *tmp, float *y, uint32_t rows,
uint32_t cols, uint32_t n, size_t ldy)
{
const float *c = L->clip + 4 * k;
clamp_into(tmp, x, (size_t)n * cols, c[0], c[1]);
vk_matmul(pool, L->t[k], NULL, tmp, y, rows, cols, n, ldy);
size_t ld = ldy ? ldy : rows;
for (uint32_t t = 0; t < n; t++)
clamp_into(y + t * ld, y + t * ld, rows, c[2], c[3]);
}
/* the rotary embedding of position p on half a head (len channels) */
static void rope_half(float *x, uint32_t len, float p, float theta)
{
uint32_t q = len / 2;
for (uint32_t i = 0; i < q; i++) {
float inv = 1.0f / powf(theta, (float)(2 * i) / (float)len);
float a = p * inv, cs = cosf(a), sn = sinf(a);
float x0 = x[i], x1 = x[i + q];
x[i] = x0 * cs - x1 * sn;
x[i + q] = x1 * cs + x0 * sn;
}
}
int janas_vg_encode(struct janas_vision *v, const struct janas_image *im,
int max_tokens, struct janas_pool *pool, float **out,
int *nx, int *ny, char *err, size_t err_len)
{
if (v->unified)
return janas_vgu_encode(v, im, max_tokens, pool, out, nx, ny, err,
err_len);
*out = NULL;
int rw, rh;
janas_vg_size(v, im->w, im->h, max_tokens, &rw, &rh, nx, ny);
struct janas_image rs;
const struct janas_image *src = im;
if (rw != im->w || rh != im->h) {
if (janas_image_resize(im, rw, rh, &rs) != 0) {
snprintf(err, err_len, "out of memory");
return -1;
}
src = &rs;
}
uint32_t P = v->patch, D = v->dim, K = v->pool_k, hd = D / v->heads;
uint32_t gw = (uint32_t)rw / P, gh = (uint32_t)rh / P, n = gw * gh;
uint32_t pd = 3 * P * P, wide = v->ff > 3 * D ? v->ff : 3 * D;
if (pd > wide)
wide = pd;
float *pix = malloc(sizeof(float) * (size_t)n * pd);
float *x = malloc(sizeof(float) * (size_t)n * D);
float *h = malloc(sizeof(float) * (size_t)n * D);
float *tmp = malloc(sizeof(float) * (size_t)n * wide);
float *qkv = malloc(sizeof(float) * (size_t)n * 3 * D);
float *g = malloc(sizeof(float) * (size_t)n * v->ff);
float *u = malloc(sizeof(float) * (size_t)n * v->ff);
float *buf = malloc(sizeof(float) * (size_t)n * D * 2);
float *res = NULL;
struct janas_pool *own = NULL;
int rc = -1;
if (!pool && !(pool = own = janas_pool_create(1, NULL))) {
snprintf(err, err_len, "out of memory");
goto done;
}
if (!pix || !x || !h || !tmp || !qkv || !g || !u || !buf) {
snprintf(err, err_len, "out of memory");
goto done;
}
/* patches in rows, each (row, column, channel), pixels in [-1, 1] */
for (uint32_t r = 0; r < gh; r++)
for (uint32_t c = 0; c < gw; c++) {
float *p = pix + ((size_t)r * gw + c) * pd;
for (uint32_t yy = 0; yy < P; yy++)
for (uint32_t xx = 0; xx < P; xx++)
for (uint32_t ch = 0; ch < 3; ch++)
*p++ = src->rgb[((size_t)(r * P + yy) * (size_t)rw +
(c * P + xx)) *
3 +
ch] *
(2.0f / 255.0f) -
1.0f;
}
vk_matmul(pool, v->t_g_patch, NULL, pix, x, D, pd, n, 0);
for (uint32_t r = 0; r < gh; r++)
for (uint32_t c = 0; c < gw; c++) {
float *xi = x + ((size_t)r * gw + c) * D;
const uint16_t *ex =
v->g_pos + (size_t)(c < v->pos_size ? c : 0) * D;
const uint16_t *ey =
v->g_pos +
((size_t)v->pos_size + (r < v->pos_size ? r : 0)) * D;
for (uint32_t k = 0; k < D; k++)
xi[k] += vk_bf(ex[k]) + vk_bf(ey[k]);
}
for (uint32_t l = 0; l < v->n_block; l++) {
const struct glayer *L = &v->gl[l];
for (uint32_t i = 0; i < n; i++)
vk_rms_norm(h + (size_t)i * D, x + (size_t)i * D, L->n_in, D,
v->eps);
clipped(pool, L, GL_Q, h, tmp, qkv, D, D, n, 3 * (size_t)D);
clipped(pool, L, GL_K, h, tmp, qkv + D, D, D, n, 3 * (size_t)D);
clipped(pool, L, GL_V, h, tmp, qkv + 2 * D, D, D, n, 3 * (size_t)D);
/* per head: queries and keys normed and rotated, values normed */
for (uint32_t i = 0; i < n; i++) {
float col = (float)(i % gw), row = (float)(i / gw);
float *q = qkv + (size_t)i * 3 * D;
for (uint32_t hh = 0; hh < v->heads; hh++) {
float *qh = q + hh * hd, *kh = q + D + hh * hd;
float *vh = q + 2 * D + hh * hd;
vk_rms_norm(qh, qh, L->q_norm, hd, v->eps);
vk_rms_norm(kh, kh, L->k_norm, hd, v->eps);
vk_rms_norm(vh, vh, NULL, hd, v->eps);
rope_half(qh, hd / 2, col, v->theta);
rope_half(qh + hd / 2, hd / 2, row, v->theta);
rope_half(kh, hd / 2, col, v->theta);
rope_half(kh + hd / 2, hd / 2, row, v->theta);
}
}
vk_attention(pool, qkv, buf, h, n, D, v->heads, 1.0f);
clipped(pool, L, GL_O, h, tmp, g, D, D, n, 0);
for (uint32_t i = 0; i < n; i++) {
float *o = g + (size_t)i * D;
vk_rms_norm(o, o, L->n_post_att, D, v->eps);
for (uint32_t k = 0; k < D; k++)
x[(size_t)i * D + k] += o[k];
}
for (uint32_t i = 0; i < n; i++)
vk_rms_norm(h + (size_t)i * D, x + (size_t)i * D, L->n_pre_ff, D,
v->eps);
clipped(pool, L, GL_GATE, h, tmp, g, v->ff, D, n, 0);
clipped(pool, L, GL_UP, h, tmp, u, v->ff, D, n, 0);
for (size_t k = 0; k < (size_t)n * v->ff; k++)
g[k] = vk_gelu_tanh(g[k]) * u[k];
clipped(pool, L, GL_DOWN, g, tmp, h, D, v->ff, n, 0);
for (uint32_t i = 0; i < n; i++) {
float *o = h + (size_t)i * D;
vk_rms_norm(o, o, L->n_post_ff, D, v->eps);
for (uint32_t k = 0; k < D; k++)
x[(size_t)i * D + k] += o[k];
}
}
/* the patches averaged K x K, row-major over the pooled grid */
uint32_t pw2 = gw / K, ph2 = gh / K, m = pw2 * ph2;
memset(h, 0, sizeof(float) * (size_t)m * D);
for (uint32_t r = 0; r < gh; r++)
for (uint32_t c = 0; c < gw; c++) {
float *o = h + ((size_t)(r / K) * pw2 + c / K) * D;
const float *xi = x + ((size_t)r * gw + c) * D;
for (uint32_t k = 0; k < D; k++)
o[k] += xi[k];
}
float scale = sqrtf((float)D) / (float)(K * K);
for (uint32_t i = 0; i < m; i++) {
float *o = h + (size_t)i * D;
for (uint32_t k = 0; k < D; k++) {
o[k] *= scale;
if (v->std_bias)
o[k] = (o[k] - v->std_bias[k]) * v->std_scale[k];
}
vk_rms_norm(o, o, NULL, D, v->eps);
}
res = malloc(sizeof(float) * (size_t)m * v->proj);
if (!res) {
snprintf(err, err_len, "out of memory");
goto done;
}
vk_matmul(pool, v->t_g_proj, NULL, h, res, v->proj, D, m, 0);
*nx = (int)pw2;
*ny = (int)ph2;
*out = res;
res = NULL;
rc = 0;
done:
free(res);
free(pix);
free(x);
free(h);
free(tmp);
free(qkv);
free(g);
free(u);
free(buf);
if (own)
janas_pool_destroy(own);
if (src == &rs)
janas_image_free(&rs);
return rc;
}