37 tok/s from Qwen3-Next-80B (48 GB) on a 32 GB laptop, 24 without drafts vs 2.8 for llama.cpp; reading 62 vs 16 tok/s. Faster than llama.cpp on all 13 models compared in Q4_K_M. LLMs bigger than your RAM, pure C, CPU or GPU.
C avx2ccpu-inferenceggufinference-enginelinuxllmlocal-llmmixture-of-expertsmoenvmequantizationqwen3simdspeculative-decodingvulkan
/* SPDX-License-Identifier: GPL-3.0-or-later
Copyright (C) 2026 Maurizio Cammalleri */
#version 450
// A Gated DeltaNet layer's beta and decay for a short block (a chain of
// drafts): the dot product of row r of w (f32, 2 nv x cols: the nv betas'
// rows, then the nv alphas') with token t's normalized input x (cols
// floats), then for a beta's row sigmoid of it, for an alpha's
// exp(softplus(it + dt[h]) A[h]) - what gdn_rec.comp takes (y: per token
// the nv betas then the nv decays). dense_mm.comp's tiles of 64 rows by 64
// tokens leave such a block one workgroup, which walks the row 16 columns
// a barrier: 25 ms a pass of 8 tokens on Qwen3.5-9B (13 Oct 2026). Here a
// workgroup takes RW rows and up to NT tokens (y), each thread every 256th
// column of them for all the tokens, the sums then added across the
// workgroup; a token's inputs are read once for the RW rows (a workgroup a
// row read them 96 times a layer on Qwen3.8-27B: 13 ms a pass of 8
// tokens). Not the CPU's order of the sums.
#extension GL_EXT_buffer_reference : require
#extension GL_EXT_buffer_reference_uvec2 : require
#extension GL_KHR_shader_subgroup_basic : require
#extension GL_KHR_shader_subgroup_arithmetic : require
layout(local_size_x = 256) in;
layout(constant_id = 0) const uint NV = 8; // unused
layout(constant_id = 1) const uint SG = 8; // unused
const uint NT = 8u; // tokens a workgroup
const uint RW = 4u; // rows a workgroup (the host's GBA_RW)
layout(buffer_reference, std430, buffer_reference_align = 4) readonly buffer F { float v[]; };
layout(buffer_reference, std430, buffer_reference_align = 4) writeonly buffer W { float v[]; };
// w: rows x cols; x: n rows of cols; y: n rows of ys floats; dt, A: nv
layout(push_constant) uniform P { uvec2 w, x, y, dt, a; uint rows, cols, n, ys; } p;
shared float part[RW * NT * 32u]; /* a subgroup's sums: up to 32 subgroups */
void main()
{
uint r0 = gl_WorkGroupID.x * RW, t0 = gl_WorkGroupID.y * NT, id = gl_LocalInvocationID.x;
F Wt = F(p.w), X = F(p.x);
uint nt = min(NT, p.n - t0);
float acc[RW * NT];
for (uint i = 0; i < RW * NT; i++)
acc[i] = 0.0;
for (uint c = id; c < p.cols; c += 256u) {
float xv[NT];
for (uint k = 0; k < NT; k++)
xv[k] = k < nt ? X.v[(t0 + k) * p.cols + c] : 0.0;
for (uint j = 0; j < RW; j++) {
float w = Wt.v[min(r0 + j, p.rows - 1u) * p.cols + c];
for (uint k = 0; k < NT; k++)
acc[j * NT + k] = fma(w, xv[k], acc[j * NT + k]);
}
}
for (uint i = 0; i < RW * NT; i++) {
float s = subgroupAdd(acc[i]);
if (subgroupElect())
part[i * 32u + gl_SubgroupID] = s;
}
barrier();
uint r = r0 + id / NT, k = id % NT;
if (id < RW * NT && r < p.rows && k < nt) {
float s = 0.0;
for (uint i = 0; i < gl_NumSubgroups; i++)
s += part[id * 32u + i];
uint nv = p.rows / 2u;
float y;
if (r < nv) {
y = 1.0 / (1.0 + exp(-s)); /* beta */
} else {
float al = s + F(p.dt).v[r - nv];
float sp = al > 20.0 ? al : log(1.0 + exp(al));
y = exp(sp * F(p.a).v[r - nv]); /* the decay */
}
W(p.y).v[(t0 + k) * p.ys + r] = y;
}
}