37 tok/s from Qwen3-Next-80B (48 GB) on a 32 GB laptop, 24 without drafts vs 2.8 for llama.cpp; reading 62 vs 16 tok/s. Faster than llama.cpp on all 13 models compared in Q4_K_M. LLMs bigger than your RAM, pure C, CPU or GPU.
C avx2ccpu-inferenceggufinference-enginelinuxllmlocal-llmmixture-of-expertsmoenvmequantizationqwen3simdspeculative-decodingvulkan
/* SPDX-License-Identifier: GPL-3.0-or-later
Copyright (C) 2026 Maurizio Cammalleri */
#version 450
// After gdn_conv.comp: the layer's convolution state, the last d - 1 rows
// of the inputs - the state the call starts from (the half the position
// buffer's word 1 names), then the a logged rows it replays (word 3), then
// its n rows of mixed - for the next call: after the a rows into the half
// it started from (the base, in place: a call that replays starts from the
// base), after all of them into the half word 2 names; any n, as short as
// a draft's. Then the call's rows of mixed into the log, from row word 4 on
// (none if it is all ones), the old rows read first. Apart from
// gdn_conv.comp, which reads the state and the log before. A thread a
// channel.
#extension GL_EXT_buffer_reference : require
#extension GL_EXT_buffer_reference_uvec2 : require
layout(local_size_x = 256) in;
layout(buffer_reference, std430, buffer_reference_align = 4) readonly buffer R { float v[]; };
layout(buffer_reference, std430, buffer_reference_align = 4) buffer S { float v[]; };
layout(buffer_reference, std430, buffer_reference_align = 4) readonly buffer RP { uint v[]; };
layout(push_constant) uniform P { uvec2 mixed, state, pos, log_; uint cc, d, hs, so, n; } p;
/* Row r of the call's inputs, the state's d - 1 rows first: the state,
the log's a rows, the block's n. */
float input_row(uint r, uint c, uint a, uint base)
{
uint d1 = p.d - 1u;
if (r < d1)
return S(p.state).v[base + r * p.cc + c];
if (r < d1 + a) /* the log is written below: read as writable too */
return S(p.log_).v[(r - d1) * p.cc + c];
return R(p.mixed).v[(r - d1 - a) * p.cc + c];
}
void main()
{
uint c = gl_WorkGroupID.x * 256u + gl_LocalInvocationID.x;
if (c >= p.cc)
return;
S St = S(p.state);
RP PS = RP(p.pos);
uint a = PS.v[3], lrow = PS.v[4];
uint src = PS.v[1] * p.hs + p.so, dst = PS.v[2] * p.hs + p.so;
/* both states read before either is written: the halves may be one */
float nb[7], nd[7];
for (uint k = 0; k + 1u < p.d; k++) {
nb[k] = input_row(a + k, c, a, src);
nd[k] = input_row(a + p.n + k, c, a, src);
}
if (a > 0u)
for (uint k = 0; k + 1u < p.d; k++)
St.v[src + k * p.cc + c] = nb[k];
for (uint k = 0; k + 1u < p.d; k++)
St.v[dst + k * p.cc + c] = nd[k];
if (lrow != 0xffffffffu) {
S L = S(p.log_);
for (uint t = 0; t < p.n; t++)
L.v[(lrow + t) * p.cc + c] = R(p.mixed).v[t * p.cc + c];
}
}