Skip to content

armgit / janas

Public Imported from GitHub
0

37 tok/s from Qwen3-Next-80B (48 GB) on a 32 GB laptop, 24 without drafts vs 2.8 for llama.cpp; reading 62 vs 16 tok/s. Faster than llama.cpp on all 13 models compared in Q4_K_M. LLMs bigger than your RAM, pure C, CPU or GPU.

7.2 KB
Raw Blame
/* SPDX-License-Identifier: GPL-3.0-or-later
   Copyright (C) 2026 Maurizio Cammalleri */
#version 450
// Q6_K in bit planes (JANAS_Q6_K_P, quant.h) times up to NV Q8_K vectors,
// one row per invocation, as janas_q6kp_dot_ref computes it: the high plane
// (with the scales and d) in the row's 82-byte blocks, the two bare planes
// in arrays of their own, 64 bytes a block, each packed as Q6_K's qh field.
// At level 3 (all three planes) the weight is the Q6_K one; at 2 and 1 the
// centre of the values the missing planes stand for, folded into the block
// sum as sumi * 2^shk - c * off, the same integers and float operations as
// the CPU: the results agree bit for bit. Activations shared across the
// subgroup as in q4k_matvec.comp. The blocks are 82 bytes, so every other
// one starts 2 bytes off a 4-byte boundary: a block is read as the 6
// aligned uvec4 that cover it and realigned, as in q6k_matvec.comp, and the
// bare planes (64 bytes a block, from a 4096-byte boundary) as 4 uvec4 each.
#extension GL_EXT_integer_dot_product : require
#extension GL_KHR_shader_subgroup_basic : require
#extension GL_KHR_shader_subgroup_ballot : require
#extension GL_EXT_buffer_reference : require
#extension GL_EXT_buffer_reference_uvec2 : require

layout(local_size_x = 64) in;
layout(constant_id = 0) const uint NV = 8;
layout(constant_id = 1) const uint SG = 8;
layout(constant_id = 2) const uint ID = 0;
const uint NX = (19u + SG - 1u) / SG;

layout(buffer_reference, std430, buffer_reference_align = 4) readonly buffer U1 { uint v[]; };
layout(buffer_reference, std430, buffer_reference_align = 16) readonly buffer U4 { uvec4 v[]; };
layout(buffer_reference, std430, buffer_reference_align = 4) writeonly buffer F { float v[]; };

// level 3, 2 or 1: the planes present; p1 (level >= 2) and p0 (level 3) the
// bare planes of the first row, nb * 64 bytes a row
// With ID (constant 2) a mixture's experts, the table of *_mmid.comp
// (gpu_token_moe.c) in w: 3 uvec4 an expert, (w, x), (y, nv, ys), (the
// bare planes p1, p0: the level they make), then from uvec4 nv on one a
// workgroup's z, (expert, first vector); x's vectors nb blocks apart.
layout(push_constant) uniform P { uvec2 w, x, y; uint rows, nb, nv, xs, ys, level; uvec2 p1, p0; } p;

int dp(uint a, uint b) { return dotPacked4x8EXT(int(a), int(b)); }
uvec4 bc(uvec4 xr[NX], uint k) { return subgroupBroadcast(xr[k / SG], k % SG); }

void main()
{
    uvec2 wa = p.w, px = p.x, py = p.y, a1 = p.p1, a0 = p.p0;
    uint nv = p.nv, xs = p.xs, ys = p.ys, level = p.level,
         v0 = gl_WorkGroupID.x * NV;
    if (ID != 0u) {
        U4 T = U4(p.w);
        uvec4 tl = T.v[p.nv + gl_WorkGroupID.z];
        uvec4 t0 = T.v[3u * tl.x], t1 = T.v[3u * tl.x + 1u],
              t2 = T.v[3u * tl.x + 2u];
        wa = t0.xy, px = t0.zw, py = t1.xy, a1 = t2.xy, a0 = t2.zw;
        nv = t1.z, xs = p.nb, ys = t1.w, v0 = tl.y;
        level = a0 != uvec2(0u) ? 3u : a1 != uvec2(0u) ? 2u : 1u;
    }
    U4 P1 = U4(a1), P0 = U4(a0);
    U4 X = U4(px);
    F Y = F(py);
    uint lane = gl_SubgroupInvocationID;
    /* workgroup y: 64 rows; x: a group of NV vectors, so that the groups
       of the same rows go out together and the weights the first reads
       from memory the others find in the L3 */
    uint grow = gl_WorkGroupID.y * 64u + gl_LocalInvocationID.x;
    uint row = min(grow, p.rows - 1u);
    uint xo[NV];
    for (uint v = 0; v < NV; v++)
        xo[v] = min(v0 + v, nv - 1u) * xs * 20u;
    float acc[NV];
    for (uint v = 0; v < NV; v++)
        acc[v] = 0.0;
    uint shk = level == 3u ? 1u : level == 2u ? 3u : 5u;
    int c = level == 3u ? 64 : level == 2u ? 61 : 49;
    uint off = row * p.nb * 82u;  /* bytes */
    uint pw = row * p.nb * 4u;    /* uvec4 of the bare planes */
    for (uint b = 0; b < p.nb; b++, off += 82u, pw += 4u) {
        /* the block's 20 words (80 bytes) and d, realigned from the 16-byte
           boundary s bytes below it (s even) */
        uint lo = wa.x + off;
        U4 B = U4(uvec2(lo & ~15u, wa.y + (lo < wa.x ? 1u : 0u)));
        uint s = lo & 15u;
        uint rw[25];
        for (uint k = 0; k < 6u; k++) {
            uvec4 q = B.v[k];
            rw[4u * k] = q.x;
            rw[4u * k + 1u] = q.y;
            rw[4u * k + 2u] = q.z;
            rw[4u * k + 3u] = q.w;
        }
        rw[24] = 0u;
        uint w0s = s >> 2;
        uint wd[21];
        for (uint k = 0; k < 21u; k++)
            wd[k] = (s & 2u) != 0u
                        ? (rw[w0s + k] >> 16) | (rw[w0s + k + 1u] << 16)
                        : rw[w0s + k];
        float d = unpackHalf2x16(wd[20] & 0xffffu).x;
        int sc[16];
        for (uint k = 0; k < 4u; k++)
            for (uint j = 0; j < 4u; j++)
                sc[4u * k + j] = bitfieldExtract(int(wd[16u + k]), int(8u * j), 8);
        uint w1[16], w0[16];
        for (uint k = 0; k < 4u; k++) {
            uvec4 a1 = level >= 2u ? P1.v[pw + k] : uvec4(0u);
            uvec4 a0 = level >= 3u ? P0.v[pw + k] : uvec4(0u);
            w1[4u * k] = a1.x; w1[4u * k + 1u] = a1.y; w1[4u * k + 2u] = a1.z; w1[4u * k + 3u] = a1.w;
            w0[4u * k] = a0.x; w0[4u * k + 1u] = a0.y; w0[4u * k + 2u] = a0.z; w0[4u * k + 3u] = a0.w;
        }
        for (uint v = 0; v < NV; v++) {
            uvec4 xr[NX];
            for (uint j = 0; j < NX; j++)
                xr[j] = X.v[xo[v] + min(j * SG + lane, 19u)];
            int isum = 0;
            for (uint n = 0; n < 2u; n++)
                for (uint part = 0; part < 4u; part++) {
                    /* 32 values: 2 groups of 16, each with its scale */
                    uint hsh = part * 2u;
                    uvec4 a0 = bc(xr, 8u * n + 2u * part);
                    uvec4 a1 = bc(xr, 8u * n + 2u * part + 1u);
                    uint a[8] = uint[8](a0.x, a0.y, a0.z, a0.w, a1.x, a1.y, a1.z, a1.w);
                    int dot0 = 0, dot1 = 0;
                    for (uint k = 0; k < 8u; k++) {
                        uint i = 8u * n + k;
                        uint q = (wd[i] >> hsh) & 0x03030303u;
                        if (level >= 2u)
                            q = (q << 2) | ((w1[i] >> hsh) & 0x03030303u);
                        if (level >= 3u)
                            q = (q << 2) | ((w0[i] >> hsh) & 0x03030303u);
                        if (k < 4u)
                            dot0 += dp(q, a[k]);
                        else
                            dot1 += dp(q, a[k]);
                    }
                    uint g = 8u * n + 2u * part;
                    isum += sc[g] * dot0 + sc[g + 1u] * dot1;
                }
            /* the centre of the missing planes, from the block sums */
            uvec4 bs0 = bc(xr, 16u), bs1 = bc(xr, 17u);
            uint bw[8] = uint[8](bs0.x, bs0.y, bs0.z, bs0.w, bs1.x, bs1.y, bs1.z, bs1.w);
            int boff = 0;
            for (uint s = 0; s < 8u; s++)
                boff += sc[2u * s] * bitfieldExtract(int(bw[s]), 0, 16) +
                        sc[2u * s + 1u] * bitfieldExtract(int(bw[s]), 16, 16);
            float d8 = uintBitsToFloat(bc(xr, 18u).x);
            acc[v] = fma(d * 0.5 * d8, float(isum * (1 << shk) - c * boff), acc[v]);
            xo[v] += 20u;
        }
    }
    if (grow < p.rows)
        for (uint v = 0; v < NV; v++)
            if (v0 + v < nv)
                Y.v[(v0 + v) * ys + row] = acc[v];
}