Skip to content

armgit / janas

Public Imported from GitHub
0

37 tok/s from Qwen3-Next-80B (48 GB) on a 32 GB laptop, 24 without drafts vs 2.8 for llama.cpp; reading 62 vs 16 tok/s. Faster than llama.cpp on all 13 models compared in Q4_K_M. LLMs bigger than your RAM, pure C, CPU or GPU.

5.5 KB
Raw Blame
/* SPDX-License-Identifier: GPL-3.0-or-later
   Copyright (C) 2026 Maurizio Cammalleri */
#version 450
// Q4_K weights times one Q8_K vector, a subgroup per row, eight lanes per
// block: lane j of a cluster of eight reads the block's j-th 16 bytes of
// nibbles, so that one load of the cluster is 128 contiguous bytes, and
// works out their part of the block's integer sums (sub-block j/2's two
// halves, and one of the eight minimum terms); the parts are added exactly
// (integers, a clustered add), and one lane chains the block's fmas in the
// CPU's order (janas_q4k_dot_ref): the result is the CPU's bit for bit. A
// subgroup of SG lanes takes SG / 8 blocks at a time. Up to NV vectors
// (the block's, or a speculative pass's drafts): the weights loaded once. For single vectors,
// where the products are bound by memory: q4k_matvec.comp reads a whole
// row per invocation, rows far apart at once, and draws a fraction of the
// bandwidth.
#extension GL_EXT_integer_dot_product : require
#extension GL_KHR_shader_subgroup_basic : require
#extension GL_KHR_shader_subgroup_clustered : require
#extension GL_KHR_shader_subgroup_shuffle : require
#extension GL_EXT_buffer_reference : require
#extension GL_EXT_buffer_reference_uvec2 : require

layout(local_size_x = 64) in;
layout(constant_id = 0) const uint NV = 1;
layout(constant_id = 1) const uint SG = 8;
layout(constant_id = 2) const uint TILE = 0; /* gpu.h JANAS_TILE, or 0 */
const uint RPW = 64u / SG; /* rows a workgroup: a subgroup each */
const uint G = SG / 8u;    /* blocks a subgroup takes a step */

layout(buffer_reference, std430, buffer_reference_align = 16) readonly buffer U4 { uvec4 v[]; };
layout(buffer_reference, std430, buffer_reference_align = 4) writeonly buffer F { float v[]; };

layout(push_constant) uniform P { uvec2 w, x, y; uint rows, nb, nv, xs, ys; } p;


int dp(uint a, uint b) { return dotPacked4x8EXT(int(a), int(b)); }
int dp4(uvec4 a, uvec4 b) { return dp(a.x, b.x) + dp(a.y, b.y) + dp(a.z, b.z) + dp(a.w, b.w); }

/* This lane's part of the exact integer sums of a block (header hd, its
   16 bytes q of nibbles) against the vector whose block starts at xo:
   added across its cluster of eight, with its two scale products. */
void blk(U4 X, uvec4 hd, uvec4 q, uint xo, uint j, out float sa, out float sb,
         out int isum, out int imin)
{
    /* the scale and minimum of sub-blocks 2c and 2c + 1, c = j / 2 */
    uint c = j / 2u, s0 = 2u * c, s1 = s0 + 1u;
    int sc0 = s0 < 4u ? int(bitfieldExtract(hd.y, int(8u * s0), 6))
                      : int(bitfieldExtract(hd.w, int(8u * (s0 - 4u)), 4) |
                            (bitfieldExtract(hd.y, int(8u * (s0 - 4u) + 6u), 2) << 4));
    int sc1 = s1 < 4u ? int(bitfieldExtract(hd.y, int(8u * s1), 6))
                      : int(bitfieldExtract(hd.w, int(8u * (s1 - 4u)), 4) |
                            (bitfieldExtract(hd.y, int(8u * (s1 - 4u) + 6u), 2) << 4));
    uint half_ = j & 1u;
    isum = dp4(q & 0x0f0f0f0fu, X.v[xo + 4u * c + half_]) * sc0 +
               dp4((q >> 4) & 0x0f0f0f0fu, X.v[xo + 4u * c + 2u + half_]) * sc1;
    /* minimum term j: mn[j] times the activations' sum of sub-block j */
    int mnj = j < 4u ? int(bitfieldExtract(hd.z, int(8u * j), 6))
                     : int(bitfieldExtract(hd.w, int(8u * (j - 4u) + 4u), 4) |
                           (bitfieldExtract(hd.z, int(8u * (j - 4u) + 6u), 2) << 4));
    uvec4 bs = X.v[xo + 16u + j / 4u];
    uint bw = j % 4u == 0u ? bs.x : j % 4u == 1u ? bs.y : j % 4u == 2u ? bs.z : bs.w;
    imin = mnj * (bitfieldExtract(int(bw), 0, 16) + bitfieldExtract(int(bw), 16, 16));
    isum = subgroupClusteredAdd(isum, 8u);
    imin = subgroupClusteredAdd(imin, 8u);
    float d8 = uintBitsToFloat(X.v[xo + 18u].x);
    vec2 dd = unpackHalf2x16(hd.x);
    sa = dd.x * d8;
    sb = dd.y * d8;
}

void main()
{
    U4 W = U4(p.w), X = U4(p.x);
    F Y = F(p.y);
    uint lane = gl_SubgroupInvocationID, sg = gl_SubgroupID;
    uint j = lane % 8u, g = lane / 8u;
    uint row = gl_WorkGroupID.x * RPW + sg;
    bool live = row < p.rows;
    uint r = min(row, p.rows - 1u);
    /* NV vectors from v0 (workgroup y) on, of p.nv, xs blocks apart: the
       weights read once for all */
    uint v0 = gl_WorkGroupID.y * NV;
    float acc[NV], accm[NV];
    for (uint v = 0; v < NV; v++) {
        acc[v] = 0.0;
        accm[v] = 0.0;
    }
    /* blocks b0 .. b0 + G - 1 a step, a cluster each, chained in that order */
    for (uint b0 = 0; b0 < p.nb; b0 += G) {
        uint bb = min(b0 + g, p.nb - 1u);
        uint wo = (TILE == 0u ? r * p.nb + bb : ((r / TILE) * p.nb + bb) * TILE + r % TILE) * 9u;
        uvec4 hd = W.v[wo], q = W.v[wo + 1u + j];
        for (uint v = 0; v < NV; v++) {
            uint xo = (min(v0 + v, p.nv - 1u) * p.xs + bb) * 20u;
            float sa, sb;
            int si, sj;
            blk(X, hd, q, xo, j, sa, sb, si, sj);
            for (uint k = 0; k < G; k++) {
                float ak = G == 1u ? sa : subgroupShuffle(sa, 8u * k);
                float bk = G == 1u ? sb : subgroupShuffle(sb, 8u * k);
                int ik = G == 1u ? si : subgroupShuffle(si, 8u * k);
                int jk = G == 1u ? sj : subgroupShuffle(sj, 8u * k);
                if (lane == 0u && b0 + k < p.nb) {
                    acc[v] = fma(ak, float(ik), acc[v]);
                    accm[v] = fma(bk, float(jk), accm[v]);
                }
            }
        }
    }
    if (lane == 0u && live)
        for (uint v = 0; v < NV; v++)
            if (v0 + v < p.nv)
                Y.v[(v0 + v) * p.ys + row] = acc[v] - accm[v];
}