Skip to content

armgit / janas

Public Imported from GitHub
0

37 tok/s from Qwen3-Next-80B (48 GB) on a 32 GB laptop, 24 without drafts vs 2.8 for llama.cpp; reading 62 vs 16 tok/s. Faster than llama.cpp on all 13 models compared in Q4_K_M. LLMs bigger than your RAM, pure C, CPU or GPU.

13.9 KB
Raw Blame
/* SPDX-License-Identifier: GPL-3.0-or-later
   Copyright (C) 2026 Maurizio Cammalleri */
#version 450
// Weights in blocks of 32 with four- or five-bit values, KIND chosen when
// the pipeline is made: 0 Q4_0 (q - 8), 1 IQ4_NL (a table of sixteen
// levels), 2 Q4_1 and 3 Q5_1 (d q + m, Q5_1 with a fifth bit per weight in
// qh), 4 MXFP4 (17 bytes a block: its E8M0 byte, the power of two
// 2^(e - 128) for a scale, then the nibbles, through MXFP4's table of 0 to
// 12 and their negatives), 5 Q5_0 (22 bytes: d, qh, the nibbles; q - 16),
// 6 Q2_0 (18 bytes for 64 weights: d, then two bits a weight, weight i in
// bits 2 (i % 4) of byte i / 4; q - 1: each half of a block one block of
// 32 here, with the block's d), 7 NVFP4 (36 bytes for 64 weights: four
// UE4M3 scales, one a sub-block of 16, then sub-block s's codes in bytes
// 8s to 8s + 7, low nibbles its first eight weights, high ones the next
// eight, through MXFP4's table; each half of a block one block of 32
// here, its two sub-blocks' dots each in its own fma, the first first, as
// janas_nvfp4_dot_ref). As the CPU computes them (nib_dot_ref,
// janas_mxfp4_dot_ref, dm_dot_ref, janas_q5_0_dot_ref and
// janas_q2_0_dot_ref in quant.c): each block's integer dot in one fma into
// the accumulator of its position in the group of eight, the minimum of
// Q4_1/Q5_1 (m times the block's activation sum) in accumulators of its
// own, the eight added at the end as janas_hsum8 does - the same float
// operations, so the results agree bit for bit. Q4_0's q - 8, Q5_0's q - 16
// and Q2_0's q - 1 are folded as the exact integer sum q.a - c sum(a).
// A row may end inside a group of 256 (tail, blocks of 32 past the whole
// groups: qwen4exp's widths of 640): those blocks are summed apart, from
// zero, and added to the whole groups' sum, as janas_dot_rows_n sums the
// group the CPU pads with empty blocks; the row is read no further than it
// goes, rows of whole words (gpu.c janas_gpu_can).
// The activations come in the 32-block layout (20 uvec4 a Q8_K block: qs32
// in 16, bsums32 in 2, d32 in 2), shared across the subgroup as in
// q4k_matvec.comp. The low nibbles of a block's 16 bytes are weights 0-15,
// the high ones 16-31. MXFP4's blocks start at any byte, so its words are
// realigned by the byte, the others' by the half word. An E8M0 byte of 0 or
// 1 is a subnormal scale, which a GPU may flush to zero where the CPU does
// not: such blocks (weights under 2^-125) differ from the CPU's.
// With ID (bit 8 of constant 2) a mixture's experts, the table of
// *_mmid.comp (gpu_token_moe.c) in w, as q4k_matvec.comp reads it: x's
// vectors nb blocks of 256 apart, one more with a tail.
#extension GL_EXT_integer_dot_product : require
#extension GL_KHR_shader_subgroup_basic : require
#extension GL_KHR_shader_subgroup_ballot : require
#extension GL_EXT_buffer_reference : require
#extension GL_EXT_buffer_reference_uvec2 : require

layout(local_size_x = 64) in;
layout(constant_id = 0) const uint NV = 8;
layout(constant_id = 1) const uint SG = 8;
layout(constant_id = 2) const uint KID = 0;
const uint KIND = KID & 255u;
const bool ID = (KID & 256u) != 0u;
const uint NX = (20u + SG - 1u) / SG;
/* bytes of a block of 32 weights (Q2_0: half a block), and of a group of
   eight in words */
const uint S = KIND == 7u ? 18u
             : KIND == 6u ? 9u
             : KIND == 5u ? 22u
             : KIND == 4u ? 17u
             : KIND == 3u ? 24u
             : KIND == 2u ? 20u
                          : 18u;
const uint GW = 2u * S;
const bool DM = KIND == 2u || KIND == 3u;
const bool TAB = KIND == 1u || KIND == 4u || KIND == 7u; /* a table */
const bool FIVE = KIND == 3u || KIND == 5u; /* a fifth bit in qh */
/* the constant folded out of the weights' integer dot */
const int OFF = KIND == 0u ? 8 : KIND == 5u ? 16 : KIND == 6u ? 1 : 0;

layout(buffer_reference, std430, buffer_reference_align = 4) readonly buffer U1 { uint v[]; };
layout(buffer_reference, std430, buffer_reference_align = 16) readonly buffer U4 { uvec4 v[]; };
layout(buffer_reference, std430, buffer_reference_align = 4) writeonly buffer F { float v[]; };

layout(push_constant) uniform P {
    uvec2 w, x, y;
    uint rows, nb, nv, xs, ys, level;
    uvec2 p1, p0;
    uint tail; /* blocks of 32 past the whole groups, 0 to 7 */
} p;

const int IQ4[16] = int[16](-127, -104, -83, -65, -49, -35, -22, -10,
                            1, 13, 25, 38, 53, 69, 89, 113);
const int MX[16] = int[16](0, 1, 2, 3, 4, 6, 8, 12, 0, -1, -2, -3, -4, -6, -8, -12);

/* a UE4M3 byte as NVFP4's scale, times one half (quant.c
   janas_ue4m3_half): exact */
float ue4m3_half(uint x)
{
    if (x == 0u || x == 0x7fu)
        return 0.0;
    int e = int((x >> 3) & 15u), man = int(x & 7u);
    return e == 0 ? ldexp(float(man), -10) : ldexp(float(8 + man), e - 11);
}

/* an E8M0 byte as MXFP4's scale, 2^(e - 128) (quant.c janas_e8m0_half) */
float e8m0_half(uint e)
{
    return uintBitsToFloat(e < 2u ? 0x00200000u << e : (e - 1u) << 23);
}

int dp(uint a, uint b) { return dotPacked4x8EXT(int(a), int(b)); }
int dp4(uvec4 a, uvec4 b) { return dp(a.x, b.x) + dp(a.y, b.y) + dp(a.z, b.z) + dp(a.w, b.w); }
uvec4 bc(uvec4 xr[NX], uint k) { return subgroupBroadcast(xr[k / SG], k % SG); }

/* four bits (weights 4j..4j+3) spread to bit 4 of each byte */
uint spread5(uint b)
{
    return ((b & 1u) << 4) | ((b & 2u) << 11) | ((b & 4u) << 18) | ((b & 8u) << 25);
}

/* a byte of Q2_0 (four weights of two bits) spread to a byte each */
uint spread2(uint b)
{
    uint t = (b | (b << 12)) & 0x000f000fu;
    return (t | (t << 6)) & 0x03030303u;
}

/* the table in the workgroup's shared memory: a constant array read at a
   varying index is far slower on some GPUs (iq4xs_matvec.comp) */
shared uint T[16];

/* each byte of w through the table (IQ4_NL's or MXFP4's) */
uint iq4(uint w)
{
    return T[w & 15u] | (T[(w >> 8) & 15u] << 8) | (T[(w >> 16) & 15u] << 16) |
           (T[(w >> 24) & 15u] << 24);
}

/* precise, as the accumulators below: no add fused with a product (a
   tail's blocks are single products from zero, which a compiler may see) */
float hsum8(float a[8])
{
    precise float s = ((a[0] + a[4]) + (a[2] + a[6])) + ((a[1] + a[5]) + (a[3] + a[7]));
    return s;
}

/* the word of wg at byte b, realigned by the half word */
uint hword(uint wg[48], uint b)
{
    return (b & 2u) != 0u ? (wg[b >> 2] >> 16) | (wg[(b >> 2) + 1u] << 16) : wg[b >> 2];
}

void main()
{
    if (gl_LocalInvocationID.x < 16u)
        T[gl_LocalInvocationID.x] =
            uint(KIND == 4u || KIND == 7u ? MX[gl_LocalInvocationID.x]
                                          : IQ4[gl_LocalInvocationID.x]) & 0xffu;
    barrier();
    uvec2 pw = p.w, px = p.x, py = p.y;
    uint nv = p.nv, xs = p.xs, ys = p.ys, v0 = gl_WorkGroupID.x * NV;
    if (ID) {
        U4 Tb = U4(p.w);
        uvec4 tl = Tb.v[p.nv + gl_WorkGroupID.z];
        uvec4 t0 = Tb.v[3u * tl.x], t1 = Tb.v[3u * tl.x + 1u];
        pw = t0.xy, px = t0.zw, py = t1.xy;
        nv = t1.z, xs = p.nb + (p.tail != 0u ? 1u : 0u), ys = t1.w, v0 = tl.y;
    }
    U1 W = U1(pw);
    U4 X = U4(px);
    F Y = F(py);
    uint lane = gl_SubgroupInvocationID;
    /* workgroup y: 64 rows; x: a group of NV vectors, so that the groups
       of the same rows go out together and the weights the first reads
       from memory the others find in the L3 */
    uint grow = gl_WorkGroupID.y * 64u + gl_LocalInvocationID.x;
    uint row = min(grow, p.rows - 1u);
    uint xo[NV];
    for (uint v = 0; v < NV; v++)
        xo[v] = min(v0 + v, nv - 1u) * xs * 20u;
    /* precise: the CPU rounds every product and every sum apart; without
       it a compiler may fuse a sum with a product before it (the tail's
       accumulators start from zero in the loop, a known value: AMD's,
       issue #46) */
    precise float acc[NV][8], accm[NV][8], res[NV];
    for (uint v = 0; v < NV; v++) {
        res[v] = 0.0;
        for (uint k = 0; k < 8u; k++) {
            acc[v][k] = 0.0;
            accm[v][k] = 0.0;
        }
    }
    /* words of the tail: whole Q2_0 blocks there, two blocks of 32 each */
    uint tw = (KIND == 6u ? p.tail / 2u * 18u : p.tail * S) / 4u;
    uint gw = row * (p.nb * GW + tw);
    /* the whole groups, then the tail (p.nb: past the last whole group) */
    for (uint b = 0; b <= p.nb; b++, gw += GW) {
        bool last = b == p.nb;
        if (last && p.tail == 0u)
            break;
        uint nk = last ? p.tail : 8u, nw = last ? tw : GW;
        uint wg[48];
        for (uint i = 0; i < GW; i++)
            wg[i] = i < nw ? W.v[gw + i] : 0u;
        float d[8], m[8];
        uvec4 lo[8], hi[8];
        for (uint k = 0; k < 8u; k++) {
            /* the block's first byte, and that of its 16 bytes of quants
               (Q2_0: of the 8 of its half) */
            uint B = KIND == 6u || KIND == 7u ? 2u * S * (k / 2u) : S * k;
            uint qb = KIND == 6u   ? B + 2u + 8u * (k % 2u)
                      : KIND == 7u ? B + 4u + 16u * (k % 2u)
                                   : B + S - 16u;
            if (KIND == 7u) { /* the half's two sub-blocks' scales */
                uint sb = B + 2u * (k % 2u);
                d[k] = ue4m3_half((wg[sb >> 2] >> (8u * (sb & 3u))) & 0xffu);
                m[k] = ue4m3_half((wg[sb >> 2] >> (8u * (sb & 3u) + 8u)) & 0xffu);
            } else if (KIND == 4u) {
                d[k] = e8m0_half((wg[B >> 2] >> (8u * (B & 3u))) & 0xffu);
                m[k] = 0.0;
            } else {
                vec2 dm = unpackHalf2x16(hword(wg, B));
                d[k] = dm.x;
                m[k] = dm.y;
            }
            uint wi = qb >> 2, sh = 8u * (qb & 3u);
            if (KIND == 6u) {
                uint q0 = sh != 0u ? (wg[wi] >> sh) | (wg[wi + 1u] << (32u - sh)) : wg[wi];
                uint q1 = sh != 0u ? (wg[wi + 1u] >> sh) | (wg[wi + 2u] << (32u - sh)) : wg[wi + 1u];
                lo[k] = uvec4(spread2(q0 & 0xffu), spread2((q0 >> 8) & 0xffu),
                              spread2((q0 >> 16) & 0xffu), spread2(q0 >> 24));
                hi[k] = uvec4(spread2(q1 & 0xffu), spread2((q1 >> 8) & 0xffu),
                              spread2((q1 >> 16) & 0xffu), spread2(q1 >> 24));
                continue;
            }
            uint q[4];
            for (uint i = 0; i < 4u; i++)
                q[i] = sh != 0u ? (wg[wi + i] >> sh) | (wg[wi + i + 1u] << (32u - sh)) : wg[wi + i];
            uint qh = KIND == 3u ? wg[(B + 4u) >> 2] : KIND == 5u ? hword(wg, B + 2u) : 0u;
            for (uint i = 0; i < 4u; i++) {
                uint l = q[i] & 0x0f0f0f0fu, h = (q[i] >> 4) & 0x0f0f0f0fu;
                if (TAB) {
                    l = iq4(l);
                    h = iq4(h);
                } else if (FIVE) {
                    l |= spread5((qh >> (4u * i)) & 15u);
                    h |= spread5((qh >> (16u + 4u * i)) & 15u);
                }
                q[i] = l;
                if (KIND == 7u) {
                    /* NVFP4: words 0-1 the first sub-block, low then high
                       nibbles; words 2-3 the second */
                    if (i == 0u) { lo[k].x = l; lo[k].z = h; }
                    if (i == 1u) { lo[k].y = l; lo[k].w = h; }
                    if (i == 2u) { hi[k].x = l; hi[k].z = h; }
                    if (i == 3u) { hi[k].y = l; hi[k].w = h; }
                    continue;
                }
                if (i == 0u) { lo[k].x = l; hi[k].x = h; }
                if (i == 1u) { lo[k].y = l; hi[k].y = h; }
                if (i == 2u) { lo[k].z = l; hi[k].z = h; }
                if (i == 3u) { lo[k].w = l; hi[k].w = h; }
            }
        }
        if (last)
            /* the whole groups' sum aside, the tail's from zero */
            for (uint v = 0; v < NV; v++) {
                if (p.nb > 0u)
                    res[v] = DM ? hsum8(acc[v]) + hsum8(accm[v]) : hsum8(acc[v]);
                for (uint k = 0; k < 8u; k++) {
                    acc[v][k] = 0.0;
                    accm[v][k] = 0.0;
                }
            }
        for (uint v = 0; v < NV; v++) {
            uvec4 xr[NX];
            for (uint j = 0; j < NX; j++)
                xr[j] = X.v[xo[v] + min(j * SG + lane, 19u)];
            uvec4 s0 = bc(xr, 18u), s1 = bc(xr, 19u);
            float d32[8] = float[8](uintBitsToFloat(s0.x), uintBitsToFloat(s0.y),
                                    uintBitsToFloat(s0.z), uintBitsToFloat(s0.w),
                                    uintBitsToFloat(s1.x), uintBitsToFloat(s1.y),
                                    uintBitsToFloat(s1.z), uintBitsToFloat(s1.w));
            uvec4 b0 = bc(xr, 16u), b1 = bc(xr, 17u);
            uint bw[8] = uint[8](b0.x, b0.y, b0.z, b0.w, b1.x, b1.y, b1.z, b1.w);
            for (uint k = 0; k < 8u; k++) {
                if (KIND == 7u) {
                    if (k < nk) {
                        acc[v][k] = fma(d[k] * d32[k], float(dp4(lo[k], bc(xr, 2u * k))),
                                        acc[v][k]);
                        acc[v][k] = fma(m[k] * d32[k], float(dp4(hi[k], bc(xr, 2u * k + 1u))),
                                        acc[v][k]);
                    }
                    continue;
                }
                int dot = dp4(lo[k], bc(xr, 2u * k)) + dp4(hi[k], bc(xr, 2u * k + 1u));
                /* the block's activation sum: bsums32 of its two halves */
                int asum = bitfieldExtract(int(bw[k]), 0, 16) + bitfieldExtract(int(bw[k]), 16, 16);
                if (OFF != 0)
                    dot -= OFF * asum;
                /* the tail's empty blocks: the CPU's zero blocks add 0 */
                if (k < nk) {
                    acc[v][k] = fma(d[k] * d32[k], float(dot), acc[v][k]);
                    if (DM)
                        accm[v][k] = fma(m[k] * d32[k], float(asum), accm[v][k]);
                }
            }
            xo[v] += 20u;
        }
    }
    if (grow < p.rows)
        for (uint v = 0; v < NV; v++)
            if (v0 + v < nv) {
                precise float r = hsum8(acc[v]);
                if (DM)
                    r = r + hsum8(accm[v]);
                if (p.tail != 0u)
                    r = res[v] + r;
                Y.v[(v0 + v) * ys + row] = r;
            }
}