37 tok/s from Qwen3-Next-80B (48 GB) on a 32 GB laptop, 24 without drafts vs 2.8 for llama.cpp; reading 62 vs 16 tok/s. Faster than llama.cpp on all 13 models compared in Q4_K_M. LLMs bigger than your RAM, pure C, CPU or GPU.
C avx2ccpu-inferenceggufinference-enginelinuxllmlocal-llmmixture-of-expertsmoenvmequantizationqwen3simdspeculative-decodingvulkan
/* SPDX-License-Identifier: GPL-3.0-or-later
Copyright (C) 2026 Maurizio Cammalleri */
#version 450
// Q6_K in bit planes (JANAS_Q6_K_P, quant.h) times up to NV Q8_K vectors,
// one row per invocation, as janas_q6kp_dot_ref computes it: the high plane
// (with the scales and d) in the row's 82-byte blocks, the two bare planes
// in arrays of their own, 64 bytes a block, each packed as Q6_K's qh field.
// At level 3 (all three planes) the weight is the Q6_K one; at 2 and 1 the
// centre of the values the missing planes stand for, folded into the block
// sum as sumi * 2^shk - c * off, the same integers and float operations as
// the CPU: the results agree bit for bit. Activations shared across the
// subgroup as in q4k_matvec.comp. The blocks are 82 bytes, so every other
// one starts 2 bytes off a 4-byte boundary: a block is read as the 6
// aligned uvec4 that cover it and realigned, as in q6k_matvec.comp, and the
// bare planes (64 bytes a block, from a 4096-byte boundary) as 4 uvec4 each.
#extension GL_EXT_integer_dot_product : require
#extension GL_KHR_shader_subgroup_basic : require
#extension GL_KHR_shader_subgroup_ballot : require
#extension GL_EXT_buffer_reference : require
#extension GL_EXT_buffer_reference_uvec2 : require
layout(local_size_x = 64) in;
layout(constant_id = 0) const uint NV = 8;
layout(constant_id = 1) const uint SG = 8;
layout(constant_id = 2) const uint ID = 0;
const uint NX = (19u + SG - 1u) / SG;
layout(buffer_reference, std430, buffer_reference_align = 4) readonly buffer U1 { uint v[]; };
layout(buffer_reference, std430, buffer_reference_align = 16) readonly buffer U4 { uvec4 v[]; };
layout(buffer_reference, std430, buffer_reference_align = 4) writeonly buffer F { float v[]; };
// level 3, 2 or 1: the planes present; p1 (level >= 2) and p0 (level 3) the
// bare planes of the first row, nb * 64 bytes a row
// With ID (constant 2) a mixture's experts, the table of *_mmid.comp
// (gpu_token_moe.c) in w: 3 uvec4 an expert, (w, x), (y, nv, ys), (the
// bare planes p1, p0: the level they make), then from uvec4 nv on one a
// workgroup's z, (expert, first vector); x's vectors nb blocks apart.
layout(push_constant) uniform P { uvec2 w, x, y; uint rows, nb, nv, xs, ys, level; uvec2 p1, p0; } p;
int dp(uint a, uint b) { return dotPacked4x8EXT(int(a), int(b)); }
uvec4 bc(uvec4 xr[NX], uint k) { return subgroupBroadcast(xr[k / SG], k % SG); }
void main()
{
uvec2 wa = p.w, px = p.x, py = p.y, a1 = p.p1, a0 = p.p0;
uint nv = p.nv, xs = p.xs, ys = p.ys, level = p.level,
v0 = gl_WorkGroupID.x * NV;
if (ID != 0u) {
U4 T = U4(p.w);
uvec4 tl = T.v[p.nv + gl_WorkGroupID.z];
uvec4 t0 = T.v[3u * tl.x], t1 = T.v[3u * tl.x + 1u],
t2 = T.v[3u * tl.x + 2u];
wa = t0.xy, px = t0.zw, py = t1.xy, a1 = t2.xy, a0 = t2.zw;
nv = t1.z, xs = p.nb, ys = t1.w, v0 = tl.y;
level = a0 != uvec2(0u) ? 3u : a1 != uvec2(0u) ? 2u : 1u;
}
U4 P1 = U4(a1), P0 = U4(a0);
U4 X = U4(px);
F Y = F(py);
uint lane = gl_SubgroupInvocationID;
/* workgroup y: 64 rows; x: a group of NV vectors, so that the groups
of the same rows go out together and the weights the first reads
from memory the others find in the L3 */
uint grow = gl_WorkGroupID.y * 64u + gl_LocalInvocationID.x;
uint row = min(grow, p.rows - 1u);
uint xo[NV];
for (uint v = 0; v < NV; v++)
xo[v] = min(v0 + v, nv - 1u) * xs * 20u;
float acc[NV];
for (uint v = 0; v < NV; v++)
acc[v] = 0.0;
uint shk = level == 3u ? 1u : level == 2u ? 3u : 5u;
int c = level == 3u ? 64 : level == 2u ? 61 : 49;
uint off = row * p.nb * 82u; /* bytes */
uint pw = row * p.nb * 4u; /* uvec4 of the bare planes */
for (uint b = 0; b < p.nb; b++, off += 82u, pw += 4u) {
/* the block's 20 words (80 bytes) and d, realigned from the 16-byte
boundary s bytes below it (s even) */
uint lo = wa.x + off;
U4 B = U4(uvec2(lo & ~15u, wa.y + (lo < wa.x ? 1u : 0u)));
uint s = lo & 15u;
uint rw[25];
for (uint k = 0; k < 6u; k++) {
uvec4 q = B.v[k];
rw[4u * k] = q.x;
rw[4u * k + 1u] = q.y;
rw[4u * k + 2u] = q.z;
rw[4u * k + 3u] = q.w;
}
rw[24] = 0u;
uint w0s = s >> 2;
uint wd[21];
for (uint k = 0; k < 21u; k++)
wd[k] = (s & 2u) != 0u
? (rw[w0s + k] >> 16) | (rw[w0s + k + 1u] << 16)
: rw[w0s + k];
float d = unpackHalf2x16(wd[20] & 0xffffu).x;
int sc[16];
for (uint k = 0; k < 4u; k++)
for (uint j = 0; j < 4u; j++)
sc[4u * k + j] = bitfieldExtract(int(wd[16u + k]), int(8u * j), 8);
uint w1[16], w0[16];
for (uint k = 0; k < 4u; k++) {
uvec4 a1 = level >= 2u ? P1.v[pw + k] : uvec4(0u);
uvec4 a0 = level >= 3u ? P0.v[pw + k] : uvec4(0u);
w1[4u * k] = a1.x; w1[4u * k + 1u] = a1.y; w1[4u * k + 2u] = a1.z; w1[4u * k + 3u] = a1.w;
w0[4u * k] = a0.x; w0[4u * k + 1u] = a0.y; w0[4u * k + 2u] = a0.z; w0[4u * k + 3u] = a0.w;
}
for (uint v = 0; v < NV; v++) {
uvec4 xr[NX];
for (uint j = 0; j < NX; j++)
xr[j] = X.v[xo[v] + min(j * SG + lane, 19u)];
int isum = 0;
for (uint n = 0; n < 2u; n++)
for (uint part = 0; part < 4u; part++) {
/* 32 values: 2 groups of 16, each with its scale */
uint hsh = part * 2u;
uvec4 a0 = bc(xr, 8u * n + 2u * part);
uvec4 a1 = bc(xr, 8u * n + 2u * part + 1u);
uint a[8] = uint[8](a0.x, a0.y, a0.z, a0.w, a1.x, a1.y, a1.z, a1.w);
int dot0 = 0, dot1 = 0;
for (uint k = 0; k < 8u; k++) {
uint i = 8u * n + k;
uint q = (wd[i] >> hsh) & 0x03030303u;
if (level >= 2u)
q = (q << 2) | ((w1[i] >> hsh) & 0x03030303u);
if (level >= 3u)
q = (q << 2) | ((w0[i] >> hsh) & 0x03030303u);
if (k < 4u)
dot0 += dp(q, a[k]);
else
dot1 += dp(q, a[k]);
}
uint g = 8u * n + 2u * part;
isum += sc[g] * dot0 + sc[g + 1u] * dot1;
}
/* the centre of the missing planes, from the block sums */
uvec4 bs0 = bc(xr, 16u), bs1 = bc(xr, 17u);
uint bw[8] = uint[8](bs0.x, bs0.y, bs0.z, bs0.w, bs1.x, bs1.y, bs1.z, bs1.w);
int boff = 0;
for (uint s = 0; s < 8u; s++)
boff += sc[2u * s] * bitfieldExtract(int(bw[s]), 0, 16) +
sc[2u * s + 1u] * bitfieldExtract(int(bw[s]), 16, 16);
float d8 = uintBitsToFloat(bc(xr, 18u).x);
acc[v] = fma(d * 0.5 * d8, float(isum * (1 << shk) - c * boff), acc[v]);
xo[v] += 20u;
}
}
if (grow < p.rows)
for (uint v = 0; v < NV; v++)
if (v0 + v < nv)
Y.v[(v0 + v) * ys + row] = acc[v];
}