Skip to content

armgit / janas

Public Imported from GitHub
0

37 tok/s from Qwen3-Next-80B (48 GB) on a 32 GB laptop, 24 without drafts vs 2.8 for llama.cpp; reading 62 vs 16 tok/s. Faster than llama.cpp on all 13 models compared in Q4_K_M. LLMs bigger than your RAM, pure C, CPU or GPU.

4.7 KB
Raw Blame
/* SPDX-License-Identifier: GPL-3.0-or-later
   Copyright (C) 2026 Maurizio Cammalleri */
#version 450
// Q5_K weights times up to NV Q8_K vectors per workgroup, one row per
// invocation: Q4_K with a fifth bit (qh: bit s of byte l is the fifth bit
// of weight 32 s + l), in the same per-block integer-sum scheme as
// q4k_matvec.comp and janas_q5k_dot_ref, so the results agree with the CPU
// bit for bit. A block is 176 bytes, 11 uvec4: d, dmin and the scales, qh
// in two, the low nibbles in eight. Activations shared across the subgroup
// as in q4k_matvec.comp.
#extension GL_EXT_integer_dot_product : require
#extension GL_KHR_shader_subgroup_basic : require
#extension GL_KHR_shader_subgroup_ballot : require
#extension GL_EXT_buffer_reference : require
#extension GL_EXT_buffer_reference_uvec2 : require

layout(local_size_x = 64) in;
layout(constant_id = 0) const uint NV = 8;
layout(constant_id = 1) const uint SG = 8;
const uint NX = (19u + SG - 1u) / SG;

layout(buffer_reference, std430, buffer_reference_align = 16) readonly buffer U4 { uvec4 v[]; };
layout(buffer_reference, std430, buffer_reference_align = 4) writeonly buffer F { float v[]; };

// w: the first row; x: vector v, block b at uvec4 (v * xs + b) * 20;
// y: row r of vector v at y[v * ys + r]
layout(push_constant) uniform P { uvec2 w, x, y; uint rows, nb, nv, xs, ys; } p;

int dp(uint a, uint b) { return dotPacked4x8EXT(int(a), int(b)); }
int dp4(uvec4 a, uvec4 b) { return dp(a.x, b.x) + dp(a.y, b.y) + dp(a.z, b.z) + dp(a.w, b.w); }
uvec4 bc(uvec4 xr[NX], uint k) { return subgroupBroadcast(xr[k / SG], k % SG); }

void main()
{
    U4 W = U4(p.w), X = U4(p.x);
    F Y = F(p.y);
    uint lane = gl_SubgroupInvocationID;
    /* workgroup y: 64 rows; x: a group of NV vectors, so that the groups
       of the same rows go out together and the weights the first reads
       from memory the others find in the L3 */
    uint grow = gl_WorkGroupID.y * 64u + gl_LocalInvocationID.x;
    uint row = min(grow, p.rows - 1u);
    uint v0 = gl_WorkGroupID.x * NV;
    uint xo[NV];
    for (uint v = 0; v < NV; v++)
        xo[v] = min(v0 + v, p.nv - 1u) * p.xs * 20u;
    float acc[NV], accm[NV];
    for (uint v = 0; v < NV; v++) { acc[v] = 0.0; accm[v] = 0.0; }
    uint wo = row * p.nb * 11u;
    for (uint b = 0; b < p.nb; b++, wo += 11u) {
        uvec4 hd = W.v[wo];
        vec2 dd = unpackHalf2x16(hd.x);
        int sc[8], mn[8];
        for (uint j = 0; j < 4u; j++) {
            sc[j] = int(bitfieldExtract(hd.y, int(8u * j), 6));
            mn[j] = int(bitfieldExtract(hd.z, int(8u * j), 6));
            sc[j + 4u] = int(bitfieldExtract(hd.w, int(8u * j), 4) |
                             (bitfieldExtract(hd.y, int(8u * j + 6u), 2) << 4));
            mn[j + 4u] = int(bitfieldExtract(hd.w, int(8u * j + 4u), 4) |
                             (bitfieldExtract(hd.z, int(8u * j + 6u), 2) << 4));
        }
        uvec4 qh0 = W.v[wo + 1u], qh1 = W.v[wo + 2u];
        uvec4 q[8];
        for (uint j = 0; j < 8u; j++)
            q[j] = W.v[wo + 3u + j];
        for (uint v = 0; v < NV; v++) {
            uvec4 xr[NX];
            for (uint j = 0; j < NX; j++)
                xr[j] = X.v[xo[v] + min(j * SG + lane, 19u)];
            int isum = 0;
            for (uint c = 0; c < 4u; c++) {
                /* sub-blocks 2c (low nibbles) and 2c + 1 (high), each 32
                   weights: their fifth bits are bits 2c and 2c + 1 of qh */
                uvec4 q0 = q[2u * c], q1 = q[2u * c + 1u];
                uint s0 = 2u * c, s1 = 2u * c + 1u;
                uvec4 lo0 = (q0 & 0x0f0f0f0fu) | (((qh0 >> s0) & 0x01010101u) << 4);
                uvec4 lo1 = (q1 & 0x0f0f0f0fu) | (((qh1 >> s0) & 0x01010101u) << 4);
                uvec4 hi0 = ((q0 >> 4) & 0x0f0f0f0fu) | (((qh0 >> s1) & 0x01010101u) << 4);
                uvec4 hi1 = ((q1 >> 4) & 0x0f0f0f0fu) | (((qh1 >> s1) & 0x01010101u) << 4);
                isum += (dp4(lo0, bc(xr, 4u * c)) + dp4(lo1, bc(xr, 4u * c + 1u))) * sc[s0] +
                        (dp4(hi0, bc(xr, 4u * c + 2u)) + dp4(hi1, bc(xr, 4u * c + 3u))) * sc[s1];
            }
            uvec4 bs0 = bc(xr, 16u), bs1 = bc(xr, 17u);
            float d8 = uintBitsToFloat(bc(xr, 18u).x);
            uint bw[8] = uint[8](bs0.x, bs0.y, bs0.z, bs0.w, bs1.x, bs1.y, bs1.z, bs1.w);
            int imin = 0;
            for (uint s = 0; s < 8u; s++)
                imin += mn[s] * (bitfieldExtract(int(bw[s]), 0, 16) + bitfieldExtract(int(bw[s]), 16, 16));
            acc[v] = fma(dd.x * d8, float(isum), acc[v]);
            accm[v] = fma(dd.y * d8, float(imin), accm[v]);
            xo[v] += 20u;
        }
    }
    if (grow < p.rows)
        for (uint v = 0; v < NV; v++)
            if (v0 + v < p.nv)
                Y.v[(v0 + v) * p.ys + row] = acc[v] - accm[v];
}