37 tok/s from Qwen3-Next-80B (48 GB) on a 32 GB laptop, 24 without drafts vs 2.8 for llama.cpp; reading 62 vs 16 tok/s. Faster than llama.cpp on all 13 models compared in Q4_K_M. LLMs bigger than your RAM, pure C, CPU or GPU.
C avx2ccpu-inferenceggufinference-enginelinuxllmlocal-llmmixture-of-expertsmoenvmequantizationqwen3simdspeculative-decodingvulkan
/* SPDX-License-Identifier: GPL-3.0-or-later
Copyright (C) 2026 Maurizio Cammalleri */
#version 450
// Q8_0 weights (blocks of 32: an fp16 scale and 32 int8) times up to NV
// vectors, one row per invocation, as janas_q8_0_dot_ref computes it: the
// activations in blocks of 32 with a scale each (qs32/d32 of the Q8_K
// block), each weight block's integer dot in one fma into the accumulator
// of its position in the group of eight, the eight added at the end as
// janas_hsum8 does, ((a0+a4)+(a2+a6)) + ((a1+a5)+(a3+a7)): the same float
// operations as the CPU, so the results agree bit for bit. The activations
// come in the 32-block layout of convert_x (20 uvec4 a Q8_K block: qs32 in
// 16, bsums32 in 2, d32 in 2), shared across the subgroup as in
// q4k_matvec.comp. A weight block is 34 bytes, so every other one starts 2
// bytes off a 4-byte boundary: its words are realigned. A row may end
// inside a group of 256 (tail, an even number of blocks of 32 past the whole
// groups, so that rows start on words): as janas_dot_rows_n, the whole
// groups' sum, then the tail's from zero, padded with zero blocks, added.
// With ID (bit 8 of constant 2) a mixture's experts (Gemma 4 26B's last
// down matrices), the table of *_mmid.comp in w as b32_matvec.comp reads
// it: x's vectors nb blocks of 256 apart, one more with a tail.
#extension GL_EXT_integer_dot_product : require
#extension GL_KHR_shader_subgroup_basic : require
#extension GL_KHR_shader_subgroup_ballot : require
#extension GL_EXT_buffer_reference : require
#extension GL_EXT_buffer_reference_uvec2 : require
layout(local_size_x = 64) in;
layout(constant_id = 0) const uint NV = 8;
layout(constant_id = 1) const uint SG = 8;
layout(constant_id = 2) const uint KID = 0;
const bool ID = (KID & 256u) != 0u;
const uint NX = (20u + SG - 1u) / SG;
layout(buffer_reference, std430, buffer_reference_align = 4) readonly buffer U1 { uint v[]; };
layout(buffer_reference, std430, buffer_reference_align = 16) readonly buffer U4 { uvec4 v[]; };
layout(buffer_reference, std430, buffer_reference_align = 4) writeonly buffer F { float v[]; };
layout(push_constant) uniform P {
uvec2 w, x, y;
uint rows, nb, nv, xs, ys, level;
uvec2 p1, p0;
uint tail; /* blocks of 32 past the whole groups: 0, 2, 4 or 6 */
} p;
int dp(uint a, uint b) { return dotPacked4x8EXT(int(a), int(b)); }
int dp4(uvec4 a, uvec4 b) { return dp(a.x, b.x) + dp(a.y, b.y) + dp(a.z, b.z) + dp(a.w, b.w); }
uvec4 bc(uvec4 xr[NX], uint k) { return subgroupBroadcast(xr[k / SG], k % SG); }
void main()
{
uvec2 pw = p.w, px = p.x, py = p.y;
uint nv = p.nv, xs = p.xs, ys = p.ys, v0 = gl_WorkGroupID.x * NV;
if (ID) {
U4 Tb = U4(p.w);
uvec4 tl = Tb.v[p.nv + gl_WorkGroupID.z];
uvec4 t0 = Tb.v[3u * tl.x], t1 = Tb.v[3u * tl.x + 1u];
pw = t0.xy, px = t0.zw, py = t1.xy;
nv = t1.z, xs = p.nb + (p.tail != 0u ? 1u : 0u), ys = t1.w, v0 = tl.y;
}
U1 W = U1(pw);
U4 X = U4(px);
F Y = F(py);
uint lane = gl_SubgroupInvocationID;
/* workgroup y: 64 rows; x: a group of NV vectors, so that the groups
of the same rows go out together and the weights the first reads
from memory the others find in the L3 */
uint grow = gl_WorkGroupID.y * 64u + gl_LocalInvocationID.x;
uint row = min(grow, p.rows - 1u);
uint xo[NV];
for (uint v = 0; v < NV; v++)
xo[v] = min(v0 + v, nv - 1u) * xs * 20u;
/* precise: the CPU rounds every product and every sum apart; without
it a compiler may fuse a sum with a product before it (the tail's
accumulators start from zero in the loop, a known value: AMD's,
issue #46) */
precise float acc[NV][8], res[NV];
for (uint v = 0; v < NV; v++) {
res[v] = 0.0;
for (uint k = 0; k < 8u; k++)
acc[v][k] = 0.0;
}
/* words: a group of eight blocks is 272 bytes, the tail 34 a block */
uint tw = p.tail * 34u / 4u;
uint gw = row * (p.nb * 68u + tw);
for (uint b = 0; b <= p.nb; b++, gw += 68u) {
bool last = b == p.nb;
if (last && p.tail == 0u)
break;
uint nk = last ? p.tail : 8u, nw = last ? tw : 68u;
uint wg[68];
for (uint i = 0; i < 68u; i++)
wg[i] = i < nw ? W.v[gw + i] : 0u;
if (last)
/* the whole groups' sum aside, the tail's from zero */
for (uint v = 0; v < NV; v++) {
if (p.nb > 0u)
res[v] = ((acc[v][0] + acc[v][4]) + (acc[v][2] + acc[v][6])) +
((acc[v][1] + acc[v][5]) + (acc[v][3] + acc[v][7]));
for (uint k = 0; k < 8u; k++)
acc[v][k] = 0.0;
}
float d[8];
uvec4 q[8][2];
for (uint k = 0; k < 8u; k++) {
uint B = 34u * k; /* the block's first byte */
uint dw = wg[B >> 2];
d[k] = unpackHalf2x16((B & 2u) != 0u ? dw >> 16 : dw & 0xffffu).x;
uint qb = B + 2u, wi = qb >> 2;
uint qw[8];
for (uint i = 0; i < 8u; i++)
qw[i] = (qb & 2u) != 0u ? (wg[wi + i] >> 16) | (wg[wi + i + 1u] << 16)
: wg[wi + i];
q[k][0] = uvec4(qw[0], qw[1], qw[2], qw[3]);
q[k][1] = uvec4(qw[4], qw[5], qw[6], qw[7]);
}
for (uint v = 0; v < NV; v++) {
uvec4 xr[NX];
for (uint j = 0; j < NX; j++)
xr[j] = X.v[xo[v] + min(j * SG + lane, 19u)];
uvec4 s0 = bc(xr, 18u), s1 = bc(xr, 19u);
float d32[8] = float[8](uintBitsToFloat(s0.x), uintBitsToFloat(s0.y),
uintBitsToFloat(s0.z), uintBitsToFloat(s0.w),
uintBitsToFloat(s1.x), uintBitsToFloat(s1.y),
uintBitsToFloat(s1.z), uintBitsToFloat(s1.w));
for (uint k = 0; k < 8u; k++) {
int dot = dp4(q[k][0], bc(xr, 2u * k)) + dp4(q[k][1], bc(xr, 2u * k + 1u));
/* the tail's empty blocks: the CPU's zero blocks add 0 */
if (k < nk)
acc[v][k] = fma(d[k] * d32[k], float(dot), acc[v][k]);
}
xo[v] += 20u;
}
}
if (grow < p.rows)
for (uint v = 0; v < NV; v++)
if (v0 + v < nv) {
precise float r = ((acc[v][0] + acc[v][4]) + (acc[v][2] + acc[v][6])) +
((acc[v][1] + acc[v][5]) + (acc[v][3] + acc[v][7]));
if (p.tail != 0u)
r = res[v] + r;
Y.v[(v0 + v) * ys + row] = r;
}
}