37 tok/s from Qwen3-Next-80B (48 GB) on a 32 GB laptop, 24 without drafts vs 2.8 for llama.cpp; reading 62 vs 16 tok/s. Faster than llama.cpp on all 13 models compared in Q4_K_M. LLMs bigger than your RAM, pure C, CPU or GPU.
C avx2ccpu-inferenceggufinference-enginelinuxllmlocal-llmmixture-of-expertsmoenvmequantizationqwen3simdspeculative-decodingvulkan
/* SPDX-License-Identifier: GPL-3.0-or-later
Copyright (C) 2026 Maurizio Cammalleri */
#version 450
// After the q, k, v products of one token (qkv: q, then k at qd, then v at
// qd + kvd floats): each head of q normalized (q_norm, unless flags' bit
// 0 says the model has none: llama; k's bit 1) and turned (NeoX
// RoPE over its first n_rot dimensions, pairs i and i + n_rot/2, the
// token's tables cs and sn), in place; each head of k normalized (k_norm)
// and turned, then k and v put in the layer's KV cache at eight bits with
// a scale each (the largest of the head's numbers becomes 127), row `row`
// of the head's positions, as model_attn.c's qkv_item does. Each head of q
// is left at eight bits too, for attn_decode.comp: its hd / 4 words of four
// bytes, then its scale (the largest magnitude over 127), at the start of
// its own hd floats; with flags' bit 2 at sixteen bits, as two halves of
// eight (q = 256 hi + lo, each -128 .. 127): hd / 4 words of hi, hd / 4 of
// lo, then the scale. Gemma 4 (flags' bits 3 to 6): keys and values at
// sixteen bits (the largest becomes 32767, two numbers a word) with q left
// in float, in place; the cache a ring (row (pos + j) mod cap); V read
// from K's place, before K's norm; V normalized without weights. One
// workgroup per head: the n_head of q, then the n_kv of k (with v).
#extension GL_EXT_buffer_reference : require
#extension GL_KHR_shader_subgroup_basic : require
#extension GL_KHR_shader_subgroup_arithmetic : require
#extension GL_EXT_buffer_reference_uvec2 : require
layout(local_size_x = 128) in;
layout(buffer_reference, std430, buffer_reference_align = 4) buffer F { float v[]; };
layout(buffer_reference, std430, buffer_reference_align = 4) writeonly buffer U { uint v[]; };
layout(buffer_reference, std430, buffer_reference_align = 4) readonly buffer R0 { uint v[]; };
// qkv; q_norm and k_norm weights (hd each); rope: cos then sin, n_rot/2
// each; kc, vc: the layer's caches (n_kv x cap x hd bytes); ks, vs: their
// scales (n_kv x cap floats)
layout(push_constant) uniform P {
uvec2 qkv, qn, kn, rope, kc, vc, ks, vs;
uint n_head, n_kv, hd, n_rot, cap, qs; /* qs: floats from a token's qkv to the next's */
float eps;
uint flags; /* bit 0: q's heads not normalized, bit 1: k's; bit 2: q
at sixteen bits; bit 3: keys and values at sixteen
bits, q in float; bit 4: a ring; bit 5: V is K; bit 6:
V normalized */
uvec2 pos; /* the token's position, read at run time: row pos */
} p;
shared float sv[512], sw[512];
shared int qk[512], qv[512];
/* A workgroup's sum and maximum: each subgroup's in one operation, then
the subgroups' results in a fixed order - two barriers, not one per
level of a tree. */
shared float sg_part[64];
float wg_sum(float s)
{
float r = subgroupAdd(s);
if (subgroupElect())
sg_part[gl_SubgroupID] = r;
barrier();
float tot = sg_part[0];
for (uint i = 1u; i < gl_NumSubgroups; i++)
tot += sg_part[i];
barrier();
return tot;
}
float wg_max(float s)
{
float r = subgroupMax(s);
if (subgroupElect())
sg_part[gl_SubgroupID] = r;
barrier();
float m = sg_part[0];
for (uint i = 1u; i < gl_NumSubgroups; i++)
m = max(m, sg_part[i]);
barrier();
return m;
}
float head_sum(float s) { return wg_sum(s); }
/* As the CPU's rms_norm and its caches' scales, bit for bit: a / b and
sqrt(a) correctly rounded (a product, then the remainder by fma), the
squares summed in two floats (some 48 bits, as its double) */
float recip_det(float d)
{
precise float y = uintBitsToFloat(0x7EF311C3u - floatBitsToUint(d));
for (int i = 0; i < 3; i++)
y = fma(y, fma(-d, y, 1.0), y);
return y;
}
float div_cr(float a, float b)
{
precise float r = recip_det(b);
precise float q = a * r;
precise float e = fma(-q, b, a);
return fma(e, r, q);
}
float sqrt_cr(float a)
{
precise float y = inversesqrt(a);
precise float s = a * y;
precise float h = 0.5 * y;
precise float r = fma(-s, s, a);
return fma(r, h, s);
}
void two_sum(inout float hi, inout float lo, float b)
{
precise float s = hi + b;
precise float bb = s - hi;
precise float e = (hi - (s - bb)) + (b - bb);
hi = s;
lo = lo + e;
}
shared float ss_hi[128], ss_lo[128];
/* 1 / sqrtf((float)(ss / n) + eps) of the head in sv (n numbers), as
rms_norm: the whole workgroup */
float rms_scale(uint n)
{
uint t = gl_LocalInvocationID.x;
float hi = 0.0, lo = 0.0;
for (uint i = t; i < n; i += 128u) {
precise float v = sv[i];
precise float sq = v * v;
precise float se = fma(v, v, -sq);
two_sum(hi, lo, sq);
lo = lo + se;
}
ss_hi[t] = hi;
ss_lo[t] = lo;
barrier();
/* the threads' two floats added in a tree, two floats all the way */
for (uint s = 64u; s > 0u; s >>= 1) {
if (t < s) {
float h = ss_hi[t], l = ss_lo[t];
two_sum(h, l, ss_hi[t + s]);
ss_hi[t] = h;
ss_lo[t] = l + ss_lo[t + s];
}
barrier();
}
hi = ss_hi[0];
lo = ss_lo[0];
barrier();
precise float fn = float(n);
precise float q0 = div_cr(hi, fn);
precise float rem = fma(-q0, fn, hi) + lo;
precise float mean = q0 + rem / fn;
return div_cr(1.0, sqrt_cr(mean + p.eps));
}
float head_max(float s) { return wg_max(s); }
void main()
{
uint t = gl_LocalInvocationID.x, h = gl_WorkGroupID.x, hd = p.hd;
F QKV = F(p.qkv), QN = F(p.qn), KN = F(p.kn), R = F(p.rope);
bool isq = h < p.n_head;
uint kh = isq ? 0u : h - p.n_head;
uint qd = p.n_head * hd, kvd = p.n_kv * hd;
/* token j of a block (workgroup y): its qkv, its RoPE tables (cos then
sin, n_rot / 2 each, n_rot floats a token), its row pos + j */
uint j = gl_WorkGroupID.y, qo = j * p.qs, ro = j * p.n_rot;
uint o = qo + (isq ? h * hd : qd + kh * hd);
for (uint i = t; i < hd; i += 128u)
sv[i] = QKV.v[o + i];
/* the same for the whole workgroup (one head): its barriers too */
if ((p.flags & (isq ? 1u : 2u)) == 0u) {
barrier(); /* sv whole */
precise float scale = rms_scale(hd);
for (uint i = t; i < hd; i += 128u) {
precise float v = sv[i] * scale;
sv[i] = v * (isq ? QN.v[i] : KN.v[i]);
}
}
barrier();
uint half_ = p.n_rot / 2u;
for (uint i = t; i < hd; i += 128u) {
/* precise: a * c - b * s rounded twice, as the CPU's (never one
fma, which a driver may make of it) */
precise float r = sv[i];
if (i < half_) {
float a = sv[i], b = sv[i + half_];
r = a * R.v[ro + i] - b * R.v[ro + half_ + i];
} else if (i < p.n_rot) {
float a = sv[i - half_], b = sv[i];
r = a * R.v[ro + half_ + i - half_] + b * R.v[ro + i - half_];
}
sw[i] = r;
}
barrier();
if (isq && (p.flags & 8u) != 0u) {
/* keys at sixteen bits: the scores over q as it is */
for (uint i = t; i < hd; i += 128u)
QKV.v[o + i] = sw[i];
return;
}
if (isq) {
/* the scores' integer dot products take q at eight bits: the
largest magnitude becomes 127, q / (m / 127) rounded to even */
float mq = 0.0;
for (uint i = t; i < hd; i += 128u)
mq = max(mq, abs(sw[i]));
mq = head_max(mq);
U QW = U(p.qkv);
if ((p.flags & 4u) != 0u) {
/* the largest magnitude becomes 127 x 256 + 127, so that hi =
floor((q + 128) / 256) and lo = q - 256 hi stay in a byte */
float d = mq / 32639.0, id = d > 0.0 ? 1.0 / d : 0.0;
uint nw = hd / 4u;
for (uint w = t; w < nw; w += 128u) {
uint wh = 0u, wl = 0u;
for (uint e = 0; e < 4u; e++) {
int q = int(clamp(roundEven(sw[4u * w + e] * id), -32639.0, 32639.0));
int hi = (q + 128) >> 8, lo = q - 256 * hi;
wh |= (uint(hi) & 0xffu) << (8u * e);
wl |= (uint(lo) & 0xffu) << (8u * e);
}
QW.v[o + w] = wh;
QW.v[o + nw + w] = wl;
}
if (t == 0u)
QKV.v[o + 2u * nw] = d;
return;
}
float d = mq / 127.0, id = d > 0.0 ? 1.0 / d : 0.0;
for (uint w = t; w < hd / 4u; w += 128u) {
uint wq = 0u;
for (uint e = 0; e < 4u; e++) {
int q = int(clamp(roundEven(sw[4u * w + e] * id), -127.0, 127.0));
wq |= (uint(q) & 0xffu) << (8u * e);
}
QW.v[o + w] = wq;
}
if (t == 0u)
QKV.v[o + hd / 4u] = d;
return;
}
/* k and v of this KV head at eight bits (or sixteen), each with its
scale; V from K's place where the layer has no V of its own */
uint ov = qo + qd + ((p.flags & 32u) != 0u ? 0u : kvd) + kh * hd;
for (uint i = t; i < hd; i += 128u)
sv[i] = QKV.v[ov + i];
if ((p.flags & 64u) != 0u) { /* V normalized, no weights */
barrier(); /* sv whole */
precise float scale = rms_scale(hd);
for (uint i = t; i < hd; i += 128u) {
precise float v = sv[i] * scale;
sv[i] = v;
}
}
float mk = 0.0, mv = 0.0;
for (uint i = t; i < hd; i += 128u) {
mk = max(mk, abs(sw[i]));
mv = max(mv, abs(sv[i]));
}
mk = head_max(mk);
mv = head_max(mv);
uint row = R0(p.pos).v[0] + j;
if ((p.flags & 16u) != 0u)
row %= p.cap;
U KC = U(p.kc), VC = U(p.vc);
if ((p.flags & 8u) != 0u) {
/* sixteen bits: the largest becomes 32767, two numbers a word */
float ik = mk > 0.0 ? div_cr(32767.0, mk) : 0.0, iv = mv > 0.0 ? div_cr(32767.0, mv) : 0.0;
uint at2 = (kh * p.cap + row) * hd / 2u; /* words */
for (uint w = t; w < hd / 2u; w += 128u) {
int k0 = int(roundEven(sw[2u * w] * ik)), k1 = int(roundEven(sw[2u * w + 1u] * ik));
int v0 = int(roundEven(sv[2u * w] * iv)), v1 = int(roundEven(sv[2u * w + 1u] * iv));
KC.v[at2 + w] = (uint(k0) & 0xffffu) | (uint(k1) << 16);
VC.v[at2 + w] = (uint(v0) & 0xffffu) | (uint(v1) << 16);
}
if (t == 0u) {
F KS = F(p.ks), VS = F(p.vs);
KS.v[kh * p.cap + row] = div_cr(mk, 32767.0);
VS.v[kh * p.cap + row] = div_cr(mv, 32767.0);
}
return;
}
float ik = mk > 0.0 ? div_cr(127.0, mk) : 0.0, iv = mv > 0.0 ? div_cr(127.0, mv) : 0.0;
for (uint i = t; i < hd; i += 128u) {
qk[i] = int(roundEven(sw[i] * ik));
qv[i] = int(roundEven(sv[i] * iv));
}
barrier();
uint at = (kh * p.cap + row) * hd; /* bytes; hd is a multiple of 4 */
for (uint w = t; w < hd / 4u; w += 128u) {
KC.v[at / 4u + w] = (uint(qk[4u * w]) & 0xffu) | ((uint(qk[4u * w + 1u]) & 0xffu) << 8) |
((uint(qk[4u * w + 2u]) & 0xffu) << 16) |
((uint(qk[4u * w + 3u]) & 0xffu) << 24);
VC.v[at / 4u + w] = (uint(qv[4u * w]) & 0xffu) | ((uint(qv[4u * w + 1u]) & 0xffu) << 8) |
((uint(qv[4u * w + 2u]) & 0xffu) << 16) |
((uint(qv[4u * w + 3u]) & 0xffu) << 24);
}
if (t == 0u) {
F KS = F(p.ks), VS = F(p.vs);
KS.v[kh * p.cap + row] = div_cr(mk, 127.0);
VS.v[kh * p.cap + row] = div_cr(mv, 127.0);
}
}