37 tok/s from Qwen3-Next-80B (48 GB) on a 32 GB laptop, 24 without drafts vs 2.8 for llama.cpp; reading 62 vs 16 tok/s. Faster than llama.cpp on all 13 models compared in Q4_K_M. LLMs bigger than your RAM, pure C, CPU or GPU.
C avx2ccpu-inferenceggufinference-enginelinuxllmlocal-llmmixture-of-expertsmoenvmequantizationqwen3simdspeculative-decodingvulkan
/* SPDX-License-Identifier: GPL-3.0-or-later
Copyright (C) 2026 Maurizio Cammalleri */
#version 450
// A mixture's routing on the GPU for a prompt's block (gpu_token.c), as
// model_ffn.c route_tok: per token, the softmax of its ne router logits,
// the k largest (the first of equals), each weight divided by the sum of
// the k (with es set, times its expert's scale: Gemma 4); logit ne, the shared expert's gate (if sh), through a sigmoid, 1
// without one. A workgroup a token, its logits in shared memory, each
// search a reduction over the 64 invocations (one invocation a token read
// some 1300 logits one after the other: 0.5 ms a layer on a single token
// on the laptop's Arc, 24 Oct 2026). NOT bit for bit with the CPU: the
// GPU's exp, the router's sums in another order.
#extension GL_EXT_buffer_reference : require
#extension GL_EXT_buffer_reference_uvec2 : require
layout(local_size_x = 64) in;
layout(buffer_reference, std430, buffer_reference_align = 4) readonly buffer F { float v[]; };
layout(buffer_reference, std430, buffer_reference_align = 4) writeonly buffer FO { float v[]; };
layout(buffer_reference, std430, buffer_reference_align = 4) writeonly buffer UO { uint v[]; };
// r: the logits, rs floats a token; pe, pw: expert and weight of pair
// j k + i; shw: the shared expert's weight of each token
layout(push_constant) uniform P { uvec2 r, pe, pw, shw, es; uint ne, rs, k, n, sh; } p;
const uint MAXK = 16u, MAXE = 512u; // k <= 16, ne <= 512
const uint NONE = 0xffffffffu;
shared float sv[MAXE];
shared float rv[64];
shared uint ri[64];
shared uint taken[MAXE / 32u];
shared uint ids[MAXK];
shared float w[MAXK];
void main()
{
uint j = gl_WorkGroupID.x, id = gl_LocalInvocationID.x;
F R = F(p.r);
uint base = j * p.rs;
float mx = R.v[base];
for (uint e = id; e < p.ne; e += 64u) {
float v = R.v[base + e];
sv[e] = v;
mx = max(mx, v);
}
if (id < MAXE / 32u)
taken[id] = 0u;
rv[id] = mx;
barrier();
for (uint s = 32u; s > 0u; s >>= 1) {
if (id < s)
rv[id] = max(rv[id], rv[id + s]);
barrier();
}
mx = rv[0];
barrier();
float sum = 0.0;
for (uint e = id; e < p.ne; e += 64u)
sum += exp(sv[e] - mx);
rv[id] = sum;
barrier();
for (uint s = 32u; s > 0u; s >>= 1) {
if (id < s)
rv[id] += rv[id + s];
barrier();
}
sum = rv[0];
barrier();
for (uint i = 0u; i < p.k; i++) {
/* each invocation's largest not taken (the first of equals: e
grows), then the largest of those, the lower index of equals */
float bv = 0.0;
uint be = NONE;
for (uint e = id; e < p.ne; e += 64u) {
if ((taken[e / 32u] & (1u << (e % 32u))) != 0u)
continue;
if (be == NONE || sv[e] > bv) {
bv = sv[e];
be = e;
}
}
rv[id] = bv;
ri[id] = be;
barrier();
for (uint s = 32u; s > 0u; s >>= 1) {
if (id < s) {
uint oe = ri[id + s], me = ri[id];
float ov = rv[id + s], mv = rv[id];
if (oe != NONE &&
(me == NONE || ov > mv || (ov == mv && oe < me))) {
rv[id] = ov;
ri[id] = oe;
}
}
barrier();
}
if (id == 0u) {
uint best = ri[0];
taken[best / 32u] |= 1u << (best % 32u);
ids[i] = best;
w[i] = exp(rv[0] - mx) / sum;
}
barrier();
}
if (id != 0u)
return;
float wsum = 0.0;
for (uint i = 0u; i < p.k; i++)
wsum += w[i];
UO PE = UO(p.pe);
FO PW = FO(p.pw);
for (uint i = 0u; i < p.k; i++) {
PE.v[j * p.k + i] = ids[i];
float wi = w[i] / wsum;
if (p.es.x != 0u || p.es.y != 0u)
wi *= F(p.es).v[ids[i]];
PW.v[j * p.k + i] = wi;
}
FO SHW = FO(p.shw);
SHW.v[j] = p.sh != 0u ? 1.0 / (1.0 + exp(-R.v[base + p.ne])) : 1.0;
}