37 tok/s from Qwen3-Next-80B (48 GB) on a 32 GB laptop, 24 without drafts vs 2.8 for llama.cpp; reading 62 vs 16 tok/s. Faster than llama.cpp on all 13 models compared in Q4_K_M. LLMs bigger than your RAM, pure C, CPU or GPU.
C avx2ccpu-inferenceggufinference-enginelinuxllmlocal-llmmixture-of-expertsmoenvmequantizationqwen3simdspeculative-decodingvulkan
/* SPDX-License-Identifier: GPL-3.0-or-later
Copyright (C) 2026 Maurizio Cammalleri */
#version 450
// dense_mm.comp's products (weights left as they are, f32 (KIND 0), bf16
// (1) or Q8_0 (2), on float vectors) for a few vectors: a workgroup a row
// and VW vectors (the second dimension), its 64 invocations each a stride
// of the columns, the sums added up in shared memory. dense_mm's tiles of
// 64 rows by 64 vectors left a router's 129 rows on one token 3
// workgroups, its 2048 columns 128 steps of a barrier each (1.1 ms a
// layer on the laptop's Arc, 24 Oct 2026). y[v * ys + r] = scale x the dot
// product of row r with vector v (vectors cols floats apart). NOT bit for
// bit with the CPU: summed in another order.
#extension GL_EXT_control_flow_attributes : require
#extension GL_EXT_buffer_reference : require
#extension GL_EXT_buffer_reference_uvec2 : require
layout(local_size_x = 64) in;
layout(constant_id = 0) const uint NV = 8; // unused
layout(constant_id = 1) const uint SG = 8; // unused
layout(constant_id = 2) const uint KIND = 0;
const uint VW = 4u;
layout(buffer_reference, std430, buffer_reference_align = 4) readonly buffer F1 { float v[]; };
layout(buffer_reference, std430, buffer_reference_align = 4) readonly buffer U1 { uint v[]; };
layout(buffer_reference, std430, buffer_reference_align = 4) writeonly buffer FO { float v[]; };
layout(push_constant) uniform P { uvec2 w, x, y; uint rows, cols, nv, ys; float scale; } p;
shared float red[64 * VW];
// weight k of row r
float wt(uint r, uint k)
{
uint i = r * p.cols + k;
if (KIND == 0u)
return F1(p.w).v[i];
if (KIND == 2u) {
/* the block's first byte: its scale, then its quants */
uint b = (r * (p.cols / 32u) + k / 32u) * 34u;
U1 W = U1(p.w);
uint dw = W.v[b >> 2];
float d = unpackHalf2x16((b & 2u) != 0u ? dw >> 16 : dw & 0xffffu).x;
uint qb = b + 2u + k % 32u;
int q = bitfieldExtract(int(W.v[qb >> 2]), int(8u * (qb & 3u)), 8);
return d * float(q);
}
uint u = U1(p.w).v[i >> 1];
return uintBitsToFloat((i & 1u) != 0u ? u & 0xffff0000u : u << 16);
}
void main()
{
F1 X = F1(p.x);
FO Y = FO(p.y);
uint id = gl_LocalInvocationID.x;
uint r = gl_WorkGroupID.x, v0 = gl_WorkGroupID.y * VW;
uint nv = min(p.nv - v0, VW);
float acc[VW];
[[unroll]] for (uint j = 0; j < VW; j++) acc[j] = 0.0;
for (uint k = id; k < p.cols; k += 64u) {
float w = wt(r, k);
[[unroll]] for (uint j = 0; j < VW; j++)
if (j < nv)
acc[j] += w * X.v[(v0 + j) * p.cols + k];
}
[[unroll]] for (uint j = 0; j < VW; j++) red[j * 64u + id] = acc[j];
barrier();
[[unroll]] for (uint s = 32u; s > 0u; s >>= 1) {
if (id < s)
[[unroll]] for (uint j = 0; j < VW; j++)
red[j * 64u + id] += red[j * 64u + id + s];
barrier();
}
if (id < nv)
Y.v[(v0 + id) * p.ys + r] = red[id * 64u] * p.scale;
}