37 tok/s from Qwen3-Next-80B (48 GB) on a 32 GB laptop, 24 without drafts vs 2.8 for llama.cpp; reading 62 vs 16 tok/s. Faster than llama.cpp on all 13 models compared in Q4_K_M. LLMs bigger than your RAM, pure C, CPU or GPU.
C avx2ccpu-inferenceggufinference-enginelinuxllmlocal-llmmixture-of-expertsmoenvmequantizationqwen3simdspeculative-decodingvulkan
/* SPDX-License-Identifier: GPL-3.0-or-later
Copyright (C) 2026 Maurizio Cammalleri */
#version 450
// Q4_K weights times one Q8_K vector, a subgroup per row, eight lanes per
// block: lane j of a cluster of eight reads the block's j-th 16 bytes of
// nibbles, so that one load of the cluster is 128 contiguous bytes, and
// works out their part of the block's integer sums (sub-block j/2's two
// halves, and one of the eight minimum terms); the parts are added exactly
// (integers, a clustered add), and one lane chains the block's fmas in the
// CPU's order (janas_q4k_dot_ref): the result is the CPU's bit for bit. A
// subgroup of SG lanes takes SG / 8 blocks at a time. Up to NV vectors
// (the block's, or a speculative pass's drafts): the weights loaded once. For single vectors,
// where the products are bound by memory: q4k_matvec.comp reads a whole
// row per invocation, rows far apart at once, and draws a fraction of the
// bandwidth.
#extension GL_EXT_integer_dot_product : require
#extension GL_KHR_shader_subgroup_basic : require
#extension GL_KHR_shader_subgroup_clustered : require
#extension GL_KHR_shader_subgroup_shuffle : require
#extension GL_EXT_buffer_reference : require
#extension GL_EXT_buffer_reference_uvec2 : require
layout(local_size_x = 64) in;
layout(constant_id = 0) const uint NV = 1;
layout(constant_id = 1) const uint SG = 8;
layout(constant_id = 2) const uint TILE = 0; /* gpu.h JANAS_TILE, or 0 */
const uint RPW = 64u / SG; /* rows a workgroup: a subgroup each */
const uint G = SG / 8u; /* blocks a subgroup takes a step */
layout(buffer_reference, std430, buffer_reference_align = 16) readonly buffer U4 { uvec4 v[]; };
layout(buffer_reference, std430, buffer_reference_align = 4) writeonly buffer F { float v[]; };
layout(push_constant) uniform P { uvec2 w, x, y; uint rows, nb, nv, xs, ys; } p;
int dp(uint a, uint b) { return dotPacked4x8EXT(int(a), int(b)); }
int dp4(uvec4 a, uvec4 b) { return dp(a.x, b.x) + dp(a.y, b.y) + dp(a.z, b.z) + dp(a.w, b.w); }
/* This lane's part of the exact integer sums of a block (header hd, its
16 bytes q of nibbles) against the vector whose block starts at xo:
added across its cluster of eight, with its two scale products. */
void blk(U4 X, uvec4 hd, uvec4 q, uint xo, uint j, out float sa, out float sb,
out int isum, out int imin)
{
/* the scale and minimum of sub-blocks 2c and 2c + 1, c = j / 2 */
uint c = j / 2u, s0 = 2u * c, s1 = s0 + 1u;
int sc0 = s0 < 4u ? int(bitfieldExtract(hd.y, int(8u * s0), 6))
: int(bitfieldExtract(hd.w, int(8u * (s0 - 4u)), 4) |
(bitfieldExtract(hd.y, int(8u * (s0 - 4u) + 6u), 2) << 4));
int sc1 = s1 < 4u ? int(bitfieldExtract(hd.y, int(8u * s1), 6))
: int(bitfieldExtract(hd.w, int(8u * (s1 - 4u)), 4) |
(bitfieldExtract(hd.y, int(8u * (s1 - 4u) + 6u), 2) << 4));
uint half_ = j & 1u;
isum = dp4(q & 0x0f0f0f0fu, X.v[xo + 4u * c + half_]) * sc0 +
dp4((q >> 4) & 0x0f0f0f0fu, X.v[xo + 4u * c + 2u + half_]) * sc1;
/* minimum term j: mn[j] times the activations' sum of sub-block j */
int mnj = j < 4u ? int(bitfieldExtract(hd.z, int(8u * j), 6))
: int(bitfieldExtract(hd.w, int(8u * (j - 4u) + 4u), 4) |
(bitfieldExtract(hd.z, int(8u * (j - 4u) + 6u), 2) << 4));
uvec4 bs = X.v[xo + 16u + j / 4u];
uint bw = j % 4u == 0u ? bs.x : j % 4u == 1u ? bs.y : j % 4u == 2u ? bs.z : bs.w;
imin = mnj * (bitfieldExtract(int(bw), 0, 16) + bitfieldExtract(int(bw), 16, 16));
isum = subgroupClusteredAdd(isum, 8u);
imin = subgroupClusteredAdd(imin, 8u);
float d8 = uintBitsToFloat(X.v[xo + 18u].x);
vec2 dd = unpackHalf2x16(hd.x);
sa = dd.x * d8;
sb = dd.y * d8;
}
void main()
{
U4 W = U4(p.w), X = U4(p.x);
F Y = F(p.y);
uint lane = gl_SubgroupInvocationID, sg = gl_SubgroupID;
uint j = lane % 8u, g = lane / 8u;
uint row = gl_WorkGroupID.x * RPW + sg;
bool live = row < p.rows;
uint r = min(row, p.rows - 1u);
/* NV vectors from v0 (workgroup y) on, of p.nv, xs blocks apart: the
weights read once for all */
uint v0 = gl_WorkGroupID.y * NV;
float acc[NV], accm[NV];
for (uint v = 0; v < NV; v++) {
acc[v] = 0.0;
accm[v] = 0.0;
}
/* blocks b0 .. b0 + G - 1 a step, a cluster each, chained in that order */
for (uint b0 = 0; b0 < p.nb; b0 += G) {
uint bb = min(b0 + g, p.nb - 1u);
uint wo = (TILE == 0u ? r * p.nb + bb : ((r / TILE) * p.nb + bb) * TILE + r % TILE) * 9u;
uvec4 hd = W.v[wo], q = W.v[wo + 1u + j];
for (uint v = 0; v < NV; v++) {
uint xo = (min(v0 + v, p.nv - 1u) * p.xs + bb) * 20u;
float sa, sb;
int si, sj;
blk(X, hd, q, xo, j, sa, sb, si, sj);
for (uint k = 0; k < G; k++) {
float ak = G == 1u ? sa : subgroupShuffle(sa, 8u * k);
float bk = G == 1u ? sb : subgroupShuffle(sb, 8u * k);
int ik = G == 1u ? si : subgroupShuffle(si, 8u * k);
int jk = G == 1u ? sj : subgroupShuffle(sj, 8u * k);
if (lane == 0u && b0 + k < p.nb) {
acc[v] = fma(ak, float(ik), acc[v]);
accm[v] = fma(bk, float(jk), accm[v]);
}
}
}
}
if (lane == 0u && live)
for (uint v = 0; v < NV; v++)
if (v0 + v < p.nv)
Y.v[(v0 + v) * p.ys + row] = acc[v] - accm[v];
}