37 tok/s from Qwen3-Next-80B (48 GB) on a 32 GB laptop, 24 without drafts vs 2.8 for llama.cpp; reading 62 vs 16 tok/s. Faster than llama.cpp on all 13 models compared in Q4_K_M. LLMs bigger than your RAM, pure C, CPU or GPU.
C avx2ccpu-inferenceggufinference-enginelinuxllmlocal-llmmixture-of-expertsmoenvmequantizationqwen3simdspeculative-decodingvulkan
/* SPDX-License-Identifier: GPL-3.0-or-later
Copyright (C) 2026 Maurizio Cammalleri */
#version 450
// qwen4exp's indexer (model_qsa.c janas_m_qsa), first step for token j of a
// block (workgroup y) at position p = pos + j (pos the position buffer's
// word 0): workgroup x = h < nh normalizes query head h (rms norm with the
// weights qn) and turns its first n_rot numbers for p (NeoX halves, the
// cos and sin of row j of rope), in place; x = nh writes the token's raw
// key at half precision to row p of raw (d numbers a row). The sums in
// the GPU's order.
#extension GL_EXT_buffer_reference : require
#extension GL_EXT_buffer_reference_uvec2 : require
layout(local_size_x = 256) in;
layout(buffer_reference, std430, buffer_reference_align = 4) buffer F { float v[]; };
layout(buffer_reference, std430, buffer_reference_align = 4) readonly buffer R { float v[]; };
layout(buffer_reference, std430, buffer_reference_align = 4) writeonly buffer U { uint v[]; };
layout(buffer_reference, std430, buffer_reference_align = 4) readonly buffer RP { uint v[]; };
layout(push_constant) uniform P {
uvec2 q, k, qn, rope, raw, pos;
uint nh, d, n_rot, qs; /* qs: floats from a token's heads to the next's */
float eps;
} p;
shared float sv[256];
shared float ss[256];
void main()
{
uint t = gl_LocalInvocationID.x, h = gl_WorkGroupID.x, j = gl_WorkGroupID.y;
uint d = p.d;
if (h == p.nh) {
/* the raw key, two halves a word */
uint row = RP(p.pos).v[0] + j;
R K = R(p.k);
if (t < d / 2u)
U(p.raw).v[row * (d / 2u) + t] =
packHalf2x16(vec2(K.v[j * d + 2u * t], K.v[j * d + 2u * t + 1u]));
return;
}
F Q = F(p.q);
uint at = j * p.qs + h * d;
float x = t < d ? Q.v[at + t] : 0.0;
ss[t] = x * x;
barrier();
for (uint s = 128u; s > 0u; s >>= 1) {
if (t < s)
ss[t] += ss[t + s];
barrier();
}
float scale = 1.0 / sqrt(ss[0] / float(d) + p.eps);
if (t < d)
sv[t] = x * scale * R(p.qn).v[t];
barrier();
uint half_ = p.n_rot / 2u;
if (t < d) {
precise float v = sv[t]; /* rounded as the CPU's */
if (t < p.n_rot) {
R RO = R(p.rope);
uint i = t % half_;
float c = RO.v[j * p.n_rot + i], s = RO.v[j * p.n_rot + half_ + i];
v = t < half_ ? sv[t] * c - sv[t + half_] * s : sv[t - half_] * s + sv[t] * c;
}
Q.v[at + t] = v;
}
}