37 tok/s from Qwen3-Next-80B (48 GB) on a 32 GB laptop, 24 without drafts vs 2.8 for llama.cpp; reading 62 vs 16 tok/s. Faster than llama.cpp on all 13 models compared in Q4_K_M. LLMs bigger than your RAM, pure C, CPU or GPU.
C avx2ccpu-inferenceggufinference-enginelinuxllmlocal-llmmixture-of-expertsmoenvmequantizationqwen3simdspeculative-decodingvulkan
/* SPDX-License-Identifier: GPL-3.0-or-later
Copyright (C) 2026 Maurizio Cammalleri */
#version 450
// The experts' activation on the GPU, bit for bit as the CPU does it
// (src/llm/vmath.h janas_silu_mul_det, src/llm/quant.c
// janas_q8k_quantize_det): h = SiLU(gate) * up (gelu: GELU's tanh form,
// janas_gelu_mul_det, Gemma 4) with the deterministic e^x
// and reciprocal, then Q8_K quantization of h, written in the layout the
// product shaders read (320 bytes a block: 256 quants, 16 pair sums, d).
// KIND 1, for a down matrix in blocks of 32 (MXFP4): the layout of the
// blocks of 32 instead (norm_quant.comp's q32: eight groups of 32 quants,
// each with its own scale, the sums of 16, the eight scales), as
// janas_q8k_quantize_det makes qs32, bsums32 and d32.
// One workgroup per 256-value block; gate and up of block b of item i at
// g[(i * nb + b) * 256 ...], u likewise; the output block at x.
#extension GL_EXT_buffer_reference : require
#extension GL_KHR_shader_subgroup_basic : require
#extension GL_KHR_shader_subgroup_arithmetic : require
#extension GL_KHR_shader_subgroup_clustered : require
#extension GL_EXT_buffer_reference_uvec2 : require
layout(local_size_x = 256) in;
layout(constant_id = 2) const uint KIND = 0;
layout(buffer_reference, std430, buffer_reference_align = 4) readonly buffer F { float v[]; };
layout(buffer_reference, std430, buffer_reference_align = 4) writeonly buffer U { uint v[]; };
layout(push_constant) uniform P { uvec2 g, u, x; uint blocks, gelu; } p;
/* The workgroup's maximum: each subgroup's in one operation, then the
subgroups' - two barriers, not one per level of a tree. */
shared float sg_part[64];
float wg_max(float s)
{
float r = subgroupMax(s);
if (subgroupElect())
sg_part[gl_SubgroupID] = r;
barrier();
float m = sg_part[0];
for (uint i = 1u; i < gl_NumSubgroups; i++)
m = max(m, sg_part[i]);
barrier();
return m;
}
shared int q_s[256];
shared float m8[32]; /* KIND 1: the maxima of eight values */
shared int s8[32]; /* ... and the sums of their quants */
float exp_det(float x)
{
precise float xc = clamp(x, -88.0, 88.0);
precise float n = roundEven(xc * 1.44269504088896341);
precise float r = fma(-n, 0.693359375, xc);
r = fma(-n, -2.12194440e-4, r);
precise float y = 1.9875691500e-4;
y = fma(y, r, 1.3981999507e-3);
y = fma(y, r, 8.3334519073e-3);
y = fma(y, r, 4.1665795894e-2);
y = fma(y, r, 1.6666665459e-1);
y = fma(y, r, 5.0000001201e-1);
precise float rr = r * r;
y = fma(y, rr, r);
y = y + 1.0;
return y * uintBitsToFloat(uint(int(n) + 127) << 23);
}
float recip_det(float d)
{
precise float y = uintBitsToFloat(0x7EF311C3u - floatBitsToUint(d));
for (int i = 0; i < 3; i++) {
precise float e = fma(-d, y, 1.0);
y = fma(y, e, y);
}
return y;
}
void main()
{
uint blk = gl_WorkGroupID.x, t = gl_LocalInvocationID.x;
if (blk >= p.blocks)
return;
F G = F(p.g), Uu = F(p.u);
U X = U(p.x);
float gv = G.v[blk * 256u + t], uv = Uu.v[blk * 256u + t];
/* GELU(g) = g / (1 + e^-2z), z = sqrt(2/pi) (g + 0.044715 g^3) */
precise float z2 = gv;
if (p.gelu != 0u)
z2 = 1.5957691216057308 * fma(0.044715 * gv, gv * gv, gv);
precise float e = exp_det(clamp(-z2, -88.0, 30.0));
precise float s = recip_det(1.0 + e);
precise float gs = gv * s;
precise float h = gs * uv;
if (KIND == 1u) {
/* each group of 32 its largest magnitude and scale */
float mc = subgroupClusteredMax(abs(h), 8u);
if (t % 8u == 0u)
m8[t / 8u] = mc;
barrier();
uint l = t / 32u;
float m32 = max(max(m8[4u * l], m8[4u * l + 1u]), max(m8[4u * l + 2u], m8[4u * l + 3u]));
precise float d32 = m32 * 0.007874015718698502;
precise float id32 = d32 > 0.0 ? recip_det(d32) : 0.0;
precise float q32 = roundEven(h * id32);
int q = int(clamp(q32, -127.0, 127.0));
q_s[t] = q;
int cs = subgroupClusteredAdd(q, 8u);
if (t % 8u == 0u)
s8[t / 8u] = cs;
barrier();
uint o32 = blk * 80u;
if (t < 64u)
X.v[o32 + t] = uint(q_s[4u * t] & 0xff) | (uint(q_s[4u * t + 1u] & 0xff) << 8) |
(uint(q_s[4u * t + 2u] & 0xff) << 16) |
(uint(q_s[4u * t + 3u] & 0xff) << 24);
else if (t < 72u) {
/* two sums of 16 quants a word */
uint k = t - 64u;
int s0 = s8[4u * k] + s8[4u * k + 1u], s1 = s8[4u * k + 2u] + s8[4u * k + 3u];
X.v[o32 + t] = uint(s0 & 0xffff) | (uint(s1 & 0xffff) << 16);
} else if (t < 80u) {
uint k = t - 72u;
precise float dk =
max(max(m8[4u * k], m8[4u * k + 1u]), max(m8[4u * k + 2u], m8[4u * k + 3u])) *
0.007874015718698502;
X.v[o32 + t] = floatBitsToUint(dk);
}
return;
}
/* the block's largest magnitude (max is exact in any order) */
float amax = wg_max(abs(h));
precise float d = amax * 0.007874015718698502;
precise float id = d > 0.0 ? recip_det(d) : 0.0;
precise float qf = roundEven(h * id);
q_s[t] = int(clamp(qf, -127.0, 127.0));
barrier();
uint o = blk * 80u; /* 320 bytes = 80 words */
if (t < 64u)
X.v[o + t] = uint(q_s[4u * t] & 0xff) | (uint(q_s[4u * t + 1u] & 0xff) << 8) |
(uint(q_s[4u * t + 2u] & 0xff) << 16) |
(uint(q_s[4u * t + 3u] & 0xff) << 24);
else if (t < 72u) {
/* two pair sums per word: the sums of 16 quants */
uint k = t - 64u;
int s0 = 0, s1 = 0;
for (uint i = 0; i < 16u; i++) {
s0 += q_s[32u * k + i];
s1 += q_s[32u * k + 16u + i];
}
X.v[o + t] = uint(s0 & 0xffff) | (uint(s1 & 0xffff) << 16);
} else if (t == 72u)
X.v[o + t] = floatBitsToUint(d);
}