Skip to content

armgit / janas

Public Imported from GitHub
0

37 tok/s from Qwen3-Next-80B (48 GB) on a 32 GB laptop, 24 without drafts vs 2.8 for llama.cpp; reading 62 vs 16 tok/s. Faster than llama.cpp on all 13 models compared in Q4_K_M. LLMs bigger than your RAM, pure C, CPU or GPU.

6.5 KB
Raw Blame
/* SPDX-License-Identifier: GPL-3.0-or-later
   Copyright (C) 2026 Maurizio Cammalleri */
#version 450
// Q8_0 weights (blocks of 32: an fp16 scale and 32 int8) times up to NV
// vectors, one row per invocation, as janas_q8_0_dot_ref computes it: the
// activations in blocks of 32 with a scale each (qs32/d32 of the Q8_K
// block), each weight block's integer dot in one fma into the accumulator
// of its position in the group of eight, the eight added at the end as
// janas_hsum8 does, ((a0+a4)+(a2+a6)) + ((a1+a5)+(a3+a7)): the same float
// operations as the CPU, so the results agree bit for bit. The activations
// come in the 32-block layout of convert_x (20 uvec4 a Q8_K block: qs32 in
// 16, bsums32 in 2, d32 in 2), shared across the subgroup as in
// q4k_matvec.comp. A weight block is 34 bytes, so every other one starts 2
// bytes off a 4-byte boundary: its words are realigned. A row may end
// inside a group of 256 (tail, an even number of blocks of 32 past the whole
// groups, so that rows start on words): as janas_dot_rows_n, the whole
// groups' sum, then the tail's from zero, padded with zero blocks, added.
// With ID (bit 8 of constant 2) a mixture's experts (Gemma 4 26B's last
// down matrices), the table of *_mmid.comp in w as b32_matvec.comp reads
// it: x's vectors nb blocks of 256 apart, one more with a tail.
#extension GL_EXT_integer_dot_product : require
#extension GL_KHR_shader_subgroup_basic : require
#extension GL_KHR_shader_subgroup_ballot : require
#extension GL_EXT_buffer_reference : require
#extension GL_EXT_buffer_reference_uvec2 : require

layout(local_size_x = 64) in;
layout(constant_id = 0) const uint NV = 8;
layout(constant_id = 1) const uint SG = 8;
layout(constant_id = 2) const uint KID = 0;
const bool ID = (KID & 256u) != 0u;
const uint NX = (20u + SG - 1u) / SG;

layout(buffer_reference, std430, buffer_reference_align = 4) readonly buffer U1 { uint v[]; };
layout(buffer_reference, std430, buffer_reference_align = 16) readonly buffer U4 { uvec4 v[]; };
layout(buffer_reference, std430, buffer_reference_align = 4) writeonly buffer F { float v[]; };

layout(push_constant) uniform P {
    uvec2 w, x, y;
    uint rows, nb, nv, xs, ys, level;
    uvec2 p1, p0;
    uint tail; /* blocks of 32 past the whole groups: 0, 2, 4 or 6 */
} p;

int dp(uint a, uint b) { return dotPacked4x8EXT(int(a), int(b)); }
int dp4(uvec4 a, uvec4 b) { return dp(a.x, b.x) + dp(a.y, b.y) + dp(a.z, b.z) + dp(a.w, b.w); }
uvec4 bc(uvec4 xr[NX], uint k) { return subgroupBroadcast(xr[k / SG], k % SG); }

void main()
{
    uvec2 pw = p.w, px = p.x, py = p.y;
    uint nv = p.nv, xs = p.xs, ys = p.ys, v0 = gl_WorkGroupID.x * NV;
    if (ID) {
        U4 Tb = U4(p.w);
        uvec4 tl = Tb.v[p.nv + gl_WorkGroupID.z];
        uvec4 t0 = Tb.v[3u * tl.x], t1 = Tb.v[3u * tl.x + 1u];
        pw = t0.xy, px = t0.zw, py = t1.xy;
        nv = t1.z, xs = p.nb + (p.tail != 0u ? 1u : 0u), ys = t1.w, v0 = tl.y;
    }
    U1 W = U1(pw);
    U4 X = U4(px);
    F Y = F(py);
    uint lane = gl_SubgroupInvocationID;
    /* workgroup y: 64 rows; x: a group of NV vectors, so that the groups
       of the same rows go out together and the weights the first reads
       from memory the others find in the L3 */
    uint grow = gl_WorkGroupID.y * 64u + gl_LocalInvocationID.x;
    uint row = min(grow, p.rows - 1u);
    uint xo[NV];
    for (uint v = 0; v < NV; v++)
        xo[v] = min(v0 + v, nv - 1u) * xs * 20u;
    /* precise: the CPU rounds every product and every sum apart; without
       it a compiler may fuse a sum with a product before it (the tail's
       accumulators start from zero in the loop, a known value: AMD's,
       issue #46) */
    precise float acc[NV][8], res[NV];
    for (uint v = 0; v < NV; v++) {
        res[v] = 0.0;
        for (uint k = 0; k < 8u; k++)
            acc[v][k] = 0.0;
    }
    /* words: a group of eight blocks is 272 bytes, the tail 34 a block */
    uint tw = p.tail * 34u / 4u;
    uint gw = row * (p.nb * 68u + tw);
    for (uint b = 0; b <= p.nb; b++, gw += 68u) {
        bool last = b == p.nb;
        if (last && p.tail == 0u)
            break;
        uint nk = last ? p.tail : 8u, nw = last ? tw : 68u;
        uint wg[68];
        for (uint i = 0; i < 68u; i++)
            wg[i] = i < nw ? W.v[gw + i] : 0u;
        if (last)
            /* the whole groups' sum aside, the tail's from zero */
            for (uint v = 0; v < NV; v++) {
                if (p.nb > 0u)
                    res[v] = ((acc[v][0] + acc[v][4]) + (acc[v][2] + acc[v][6])) +
                             ((acc[v][1] + acc[v][5]) + (acc[v][3] + acc[v][7]));
                for (uint k = 0; k < 8u; k++)
                    acc[v][k] = 0.0;
            }
        float d[8];
        uvec4 q[8][2];
        for (uint k = 0; k < 8u; k++) {
            uint B = 34u * k;            /* the block's first byte */
            uint dw = wg[B >> 2];
            d[k] = unpackHalf2x16((B & 2u) != 0u ? dw >> 16 : dw & 0xffffu).x;
            uint qb = B + 2u, wi = qb >> 2;
            uint qw[8];
            for (uint i = 0; i < 8u; i++)
                qw[i] = (qb & 2u) != 0u ? (wg[wi + i] >> 16) | (wg[wi + i + 1u] << 16)
                                        : wg[wi + i];
            q[k][0] = uvec4(qw[0], qw[1], qw[2], qw[3]);
            q[k][1] = uvec4(qw[4], qw[5], qw[6], qw[7]);
        }
        for (uint v = 0; v < NV; v++) {
            uvec4 xr[NX];
            for (uint j = 0; j < NX; j++)
                xr[j] = X.v[xo[v] + min(j * SG + lane, 19u)];
            uvec4 s0 = bc(xr, 18u), s1 = bc(xr, 19u);
            float d32[8] = float[8](uintBitsToFloat(s0.x), uintBitsToFloat(s0.y),
                                    uintBitsToFloat(s0.z), uintBitsToFloat(s0.w),
                                    uintBitsToFloat(s1.x), uintBitsToFloat(s1.y),
                                    uintBitsToFloat(s1.z), uintBitsToFloat(s1.w));
            for (uint k = 0; k < 8u; k++) {
                int dot = dp4(q[k][0], bc(xr, 2u * k)) + dp4(q[k][1], bc(xr, 2u * k + 1u));
                /* the tail's empty blocks: the CPU's zero blocks add 0 */
                if (k < nk)
                    acc[v][k] = fma(d[k] * d32[k], float(dot), acc[v][k]);
            }
            xo[v] += 20u;
        }
    }
    if (grow < p.rows)
        for (uint v = 0; v < NV; v++)
            if (v0 + v < nv) {
                precise float r = ((acc[v][0] + acc[v][4]) + (acc[v][2] + acc[v][6])) +
                          ((acc[v][1] + acc[v][5]) + (acc[v][3] + acc[v][7]));
                if (p.tail != 0u)
                    r = res[v] + r;
                Y.v[(v0 + v) * ys + row] = r;
            }
}