37 tok/s from Qwen3-Next-80B (48 GB) on a 32 GB laptop, 24 without drafts vs 2.8 for llama.cpp; reading 62 vs 16 tok/s. Faster than llama.cpp on all 13 models compared in Q4_K_M. LLMs bigger than your RAM, pure C, CPU or GPU.
C avx2ccpu-inferenceggufinference-enginelinuxllmlocal-llmmixture-of-expertsmoenvmequantizationqwen3simdspeculative-decodingvulkan
/* SPDX-License-Identifier: GPL-3.0-or-later
Copyright (C) 2026 Maurizio Cammalleri */
#version 450
// A Gated DeltaNet layer's last step before its output product, as
// model_rec.c's gate_worker does: each head's raw output (ds = 128 numbers)
// normalized by its RMS and the weights w (ds), times SiLU(z), then
// quantized to Q8_K as norm_quant.comp does (q256, and q32 if set). A
// workgroup per block of 256 (two heads) and token (y), a number a thread.
// The first workgroup of each token also logs the token's beta and decay of
// each head as gdn_rec.comp used them (bd: 2 nv a token) into lb and ld
// (the layer's log, nv a row) at row (the position buffer's word 4) + y,
// where that word is not all ones.
#extension GL_EXT_buffer_reference : require
#extension GL_KHR_shader_subgroup_arithmetic : require
#extension GL_KHR_shader_subgroup_basic : require
#extension GL_KHR_shader_subgroup_clustered : require
#extension GL_EXT_buffer_reference_uvec2 : require
layout(local_size_x = 256) in;
layout(buffer_reference, std430, buffer_reference_align = 4) readonly buffer F { float v[]; };
layout(buffer_reference, std430, buffer_reference_align = 4) writeonly buffer U { uint v[]; };
layout(buffer_reference, std430, buffer_reference_align = 4) writeonly buffer W { float v[]; };
layout(buffer_reference, std430, buffer_reference_align = 4) readonly buffer RP { uint v[]; };
// att, z: n rows of d floats; w: ds floats
// sig: sigmoid(z) in place of SiLU(z) (qwen4exp)
layout(push_constant) uniform P { uvec2 att, z, w, q256, q32, bd, lb, ld, pos; uint d, nv; float eps; uint sig; } p;
shared float sq_[256];
shared float hs_[2];
shared float m8[32]; /* the block's maxima, eight values each */
shared int s8a[32], s8b[32]; /* and its quants' sums, eight each */
bool set(uvec2 a) { return a.x != 0u || a.y != 0u; }
float recip_det(float d)
{
precise float y = uintBitsToFloat(0x7EF311C3u - floatBitsToUint(d));
for (int i = 0; i < 3; i++)
y = fma(y, fma(-d, y, 1.0), y);
return y;
}
int q8(float v, float id)
{
precise float q = roundEven(v * id);
return int(clamp(q, -127.0, 127.0));
}
void main()
{
uint t = gl_LocalInvocationID.x, b = gl_WorkGroupID.x, j = gl_WorkGroupID.y;
uint e = 256u * b + t, o = j * p.d + e;
uint lrow = RP(p.pos).v[4];
if (b == 0u && t < p.nv && lrow != 0xffffffffu) {
F BD = F(p.bd);
W(p.lb).v[(lrow + j) * p.nv + t] = BD.v[j * 2u * p.nv + t];
W(p.ld).v[(lrow + j) * p.nv + t] = BD.v[j * 2u * p.nv + p.nv + t];
}
U Q = U(p.q256), Q32 = U(p.q32);
float a = F(p.att).v[o];
sq_[t] = a * a;
barrier();
if (t < 2u) { /* each head's sum of squares, in order */
float s = 0.0;
for (uint i = 0; i < 128u; i++)
s += sq_[128u * t + i];
hs_[t] = s;
}
barrier();
precise float sc = 1.0 / sqrt(hs_[t / 128u] / 128.0 + p.eps);
precise float vs = a * sc;
float z = F(p.z).v[o];
float v = vs * F(p.w).v[e % 128u] * ((p.sig != 0u ? 1.0 : z) / (1.0 + exp(-z)));
/* the maxima of the groups of 32 and of the block */
float mc = subgroupClusteredMax(abs(v), 8u);
if (t % 8u == 0u)
m8[t / 8u] = mc;
barrier();
uint l = t / 32u;
float m32 = max(max(m8[4u * l], m8[4u * l + 1u]), max(m8[4u * l + 2u], m8[4u * l + 3u]));
float m256 = 0.0;
for (uint k = 0; k < 32u; k++)
m256 = max(m256, m8[k]);
precise float dd = m256 * 0.007874015718698502;
float id = dd > 0.0 ? recip_det(dd) : 0.0;
precise float d32 = m32 * 0.007874015718698502;
float id32 = d32 > 0.0 ? recip_det(d32) : 0.0;
int qa = q8(v, id), qb = q8(v, id32);
/* four quants a word, from four neighbouring threads */
uint wa = subgroupClusteredOr((uint(qa) & 0xffu) << (8u * (t % 4u)), 4u);
uint wb = subgroupClusteredOr((uint(qb) & 0xffu) << (8u * (t % 4u)), 4u);
uint base = (j * gl_NumWorkGroups.x + b) * 80u; /* 320 bytes a block */
if (t % 4u == 0u) {
Q.v[base + t / 4u] = wa;
if (set(p.q32))
Q32.v[base + t / 4u] = wb;
}
/* the sums of 16 quants: two of eight each */
int ca = subgroupClusteredAdd(qa, 8u), cb = subgroupClusteredAdd(qb, 8u);
if (t % 8u == 0u) {
s8a[t / 8u] = ca;
s8b[t / 8u] = cb;
}
barrier();
if (t < 8u) {
int sa0 = s8a[4u * t] + s8a[4u * t + 1u], sa1 = s8a[4u * t + 2u] + s8a[4u * t + 3u];
Q.v[base + 64u + t] = (uint(sa0) & 0xffffu) | (uint(sa1) << 16);
if (set(p.q32)) {
int sb0 = s8b[4u * t] + s8b[4u * t + 1u], sb1 = s8b[4u * t + 2u] + s8b[4u * t + 3u];
Q32.v[base + 64u + t] = (uint(sb0) & 0xffffu) | (uint(sb1) << 16);
precise float dt =
max(max(m8[4u * t], m8[4u * t + 1u]), max(m8[4u * t + 2u], m8[4u * t + 3u])) *
0.007874015718698502;
Q32.v[base + 72u + t] = floatBitsToUint(dt);
}
}
if (t == 0u)
Q.v[base + 72u] = floatBitsToUint(dd);
}