37 tok/s from Qwen3-Next-80B (48 GB) on a 32 GB laptop, 24 without drafts vs 2.8 for llama.cpp; reading 62 vs 16 tok/s. Faster than llama.cpp on all 13 models compared in Q4_K_M. LLMs bigger than your RAM, pure C, CPU or GPU.
C avx2ccpu-inferenceggufinference-enginelinuxllmlocal-llmmixture-of-expertsmoenvmequantizationqwen3simdspeculative-decodingvulkan
/* SPDX-License-Identifier: GPL-3.0-or-later
Copyright (C) 2026 Maurizio Cammalleri */
#version 450
// The fast prompt for weights left as they are, f32 (KIND 0) or bf16 (1):
// the per-layer branches of Gemma 4's E2B and E4B (gpu.c
// janas_gpu_dense_mm); or Q8_0 (2: blocks of 32, an fp16 scale and 32
// int8, rows of cols / 32 blocks), on float vectors as the CPU's
// janas_q8w_dot takes them (Qwen3.8-Flash-Next's mixes rewritten by
// JANAS_HC_MIX=q8, gpu_token_hc.c). Tiles of 64 rows by 64 vectors, 16 columns staged
// per barrier, each invocation 4 rows by 4 vectors. y[v * ys + r] = scale x
// the dot product of row r with vector v (vectors cols floats apart). NOT
// bit for bit with the CPU: summed in another order.
#extension GL_EXT_control_flow_attributes : require
#extension GL_EXT_buffer_reference : require
#extension GL_EXT_buffer_reference_uvec2 : require
layout(local_size_x = 256) in;
layout(constant_id = 0) const uint NV = 8; // unused
layout(constant_id = 1) const uint SG = 8; // unused
layout(constant_id = 2) const uint KIND = 0;
const uint BM = 64u, BN = 64u, BK = 16u;
layout(buffer_reference, std430, buffer_reference_align = 4) readonly buffer F1 { float v[]; };
layout(buffer_reference, std430, buffer_reference_align = 4) readonly buffer U1 { uint v[]; };
layout(buffer_reference, std430, buffer_reference_align = 4) writeonly buffer FO { float v[]; };
layout(push_constant) uniform P { uvec2 w, x, y; uint rows, cols, nv, ys; float scale; } p;
shared float sa[BK * BM];
shared float sb[BK * BN];
// weight k of row r
float wt(uint r, uint k)
{
uint i = r * p.cols + k;
if (KIND == 0u)
return F1(p.w).v[i];
if (KIND == 2u) {
/* the block's first byte: its scale, then its quants */
uint b = (r * (p.cols / 32u) + k / 32u) * 34u;
U1 W = U1(p.w);
uint dw = W.v[b >> 2];
float d = unpackHalf2x16((b & 2u) != 0u ? dw >> 16 : dw & 0xffffu).x;
uint qb = b + 2u + k % 32u;
int q = bitfieldExtract(int(W.v[qb >> 2]), int(8u * (qb & 3u)), 8);
return d * float(q);
}
uint u = U1(p.w).v[i >> 1];
return uintBitsToFloat((i & 1u) != 0u ? u & 0xffff0000u : u << 16);
}
void main()
{
F1 X = F1(p.x);
FO Y = FO(p.y);
uint id = gl_LocalInvocationID.x;
uint r0 = gl_WorkGroupID.x * BM, v0 = gl_WorkGroupID.y * BN;
uint tr = id % 16u, tc = id / 16u;
uint ld = id / 4u, kq = (id % 4u) * 4u; // staging: a row (vector), 4 columns
uint rl = min(r0 + ld, p.rows - 1u), vl = min(v0 + ld, p.nv - 1u);
float acc[16];
[[unroll]] for (uint i = 0; i < 16u; i++) acc[i] = 0.0;
for (uint k0 = 0; k0 < p.cols; k0 += BK) {
[[unroll]] for (uint i = 0; i < 4u; i++) {
sa[(kq + i) * BM + ld] = wt(rl, k0 + kq + i);
sb[(kq + i) * BN + ld] = X.v[vl * p.cols + k0 + kq + i];
}
barrier();
[[unroll]] for (uint kk = 0; kk < BK; kk++) {
float a[4], b[4];
[[unroll]] for (uint i = 0; i < 4u; i++) {
a[i] = sa[kk * BM + tr * 4u + i];
b[i] = sb[kk * BN + tc * 4u + i];
}
[[unroll]] for (uint i = 0; i < 4u; i++)
[[unroll]] for (uint j = 0; j < 4u; j++)
acc[i * 4u + j] += a[i] * b[j];
}
barrier();
}
[[unroll]] for (uint i = 0; i < 4u; i++)
[[unroll]] for (uint j = 0; j < 4u; j++) {
uint row = r0 + tr * 4u + i, v = v0 + tc * 4u + j;
if (row < p.rows && v < p.nv)
Y.v[v * p.ys + row] = acc[i * 4u + j] * p.scale;
}
}