Skip to content

armgit / janas

Public Imported from GitHub
0

37 tok/s from Qwen3-Next-80B (48 GB) on a 32 GB laptop, 24 without drafts vs 2.8 for llama.cpp; reading 62 vs 16 tok/s. Faster than llama.cpp on all 13 models compared in Q4_K_M. LLMs bigger than your RAM, pure C, CPU or GPU.

5.8 KB
Raw Blame
/* SPDX-License-Identifier: GPL-3.0-or-later
   Copyright (C) 2026 Maurizio Cammalleri */
#version 450
// The experts' activation on the GPU, bit for bit as the CPU does it
// (src/llm/vmath.h janas_silu_mul_det, src/llm/quant.c
// janas_q8k_quantize_det): h = SiLU(gate) * up (gelu: GELU's tanh form,
// janas_gelu_mul_det, Gemma 4) with the deterministic e^x
// and reciprocal, then Q8_K quantization of h, written in the layout the
// product shaders read (320 bytes a block: 256 quants, 16 pair sums, d).
// KIND 1, for a down matrix in blocks of 32 (MXFP4): the layout of the
// blocks of 32 instead (norm_quant.comp's q32: eight groups of 32 quants,
// each with its own scale, the sums of 16, the eight scales), as
// janas_q8k_quantize_det makes qs32, bsums32 and d32.
// One workgroup per 256-value block; gate and up of block b of item i at
// g[(i * nb + b) * 256 ...], u likewise; the output block at x.
#extension GL_EXT_buffer_reference : require
#extension GL_KHR_shader_subgroup_basic : require
#extension GL_KHR_shader_subgroup_arithmetic : require
#extension GL_KHR_shader_subgroup_clustered : require
#extension GL_EXT_buffer_reference_uvec2 : require

layout(local_size_x = 256) in;
layout(constant_id = 2) const uint KIND = 0;

layout(buffer_reference, std430, buffer_reference_align = 4) readonly buffer F { float v[]; };
layout(buffer_reference, std430, buffer_reference_align = 4) writeonly buffer U { uint v[]; };

layout(push_constant) uniform P { uvec2 g, u, x; uint blocks, gelu; } p;

/* The workgroup's maximum: each subgroup's in one operation, then the
   subgroups' - two barriers, not one per level of a tree. */
shared float sg_part[64];

float wg_max(float s)
{
    float r = subgroupMax(s);
    if (subgroupElect())
        sg_part[gl_SubgroupID] = r;
    barrier();
    float m = sg_part[0];
    for (uint i = 1u; i < gl_NumSubgroups; i++)
        m = max(m, sg_part[i]);
    barrier();
    return m;
}

shared int q_s[256];
shared float m8[32]; /* KIND 1: the maxima of eight values */
shared int s8[32];   /* ... and the sums of their quants */

float exp_det(float x)
{
    precise float xc = clamp(x, -88.0, 88.0);
    precise float n = roundEven(xc * 1.44269504088896341);
    precise float r = fma(-n, 0.693359375, xc);
    r = fma(-n, -2.12194440e-4, r);
    precise float y = 1.9875691500e-4;
    y = fma(y, r, 1.3981999507e-3);
    y = fma(y, r, 8.3334519073e-3);
    y = fma(y, r, 4.1665795894e-2);
    y = fma(y, r, 1.6666665459e-1);
    y = fma(y, r, 5.0000001201e-1);
    precise float rr = r * r;
    y = fma(y, rr, r);
    y = y + 1.0;
    return y * uintBitsToFloat(uint(int(n) + 127) << 23);
}

float recip_det(float d)
{
    precise float y = uintBitsToFloat(0x7EF311C3u - floatBitsToUint(d));
    for (int i = 0; i < 3; i++) {
        precise float e = fma(-d, y, 1.0);
        y = fma(y, e, y);
    }
    return y;
}

void main()
{
    uint blk = gl_WorkGroupID.x, t = gl_LocalInvocationID.x;
    if (blk >= p.blocks)
        return;
    F G = F(p.g), Uu = F(p.u);
    U X = U(p.x);
    float gv = G.v[blk * 256u + t], uv = Uu.v[blk * 256u + t];
    /* GELU(g) = g / (1 + e^-2z), z = sqrt(2/pi) (g + 0.044715 g^3) */
    precise float z2 = gv;
    if (p.gelu != 0u)
        z2 = 1.5957691216057308 * fma(0.044715 * gv, gv * gv, gv);
    precise float e = exp_det(clamp(-z2, -88.0, 30.0));
    precise float s = recip_det(1.0 + e);
    precise float gs = gv * s;
    precise float h = gs * uv;
    if (KIND == 1u) {
        /* each group of 32 its largest magnitude and scale */
        float mc = subgroupClusteredMax(abs(h), 8u);
        if (t % 8u == 0u)
            m8[t / 8u] = mc;
        barrier();
        uint l = t / 32u;
        float m32 = max(max(m8[4u * l], m8[4u * l + 1u]), max(m8[4u * l + 2u], m8[4u * l + 3u]));
        precise float d32 = m32 * 0.007874015718698502;
        precise float id32 = d32 > 0.0 ? recip_det(d32) : 0.0;
        precise float q32 = roundEven(h * id32);
        int q = int(clamp(q32, -127.0, 127.0));
        q_s[t] = q;
        int cs = subgroupClusteredAdd(q, 8u);
        if (t % 8u == 0u)
            s8[t / 8u] = cs;
        barrier();
        uint o32 = blk * 80u;
        if (t < 64u)
            X.v[o32 + t] = uint(q_s[4u * t] & 0xff) | (uint(q_s[4u * t + 1u] & 0xff) << 8) |
                           (uint(q_s[4u * t + 2u] & 0xff) << 16) |
                           (uint(q_s[4u * t + 3u] & 0xff) << 24);
        else if (t < 72u) {
            /* two sums of 16 quants a word */
            uint k = t - 64u;
            int s0 = s8[4u * k] + s8[4u * k + 1u], s1 = s8[4u * k + 2u] + s8[4u * k + 3u];
            X.v[o32 + t] = uint(s0 & 0xffff) | (uint(s1 & 0xffff) << 16);
        } else if (t < 80u) {
            uint k = t - 72u;
            precise float dk =
                max(max(m8[4u * k], m8[4u * k + 1u]), max(m8[4u * k + 2u], m8[4u * k + 3u])) *
                0.007874015718698502;
            X.v[o32 + t] = floatBitsToUint(dk);
        }
        return;
    }
    /* the block's largest magnitude (max is exact in any order) */
    float amax = wg_max(abs(h));
    precise float d = amax * 0.007874015718698502;
    precise float id = d > 0.0 ? recip_det(d) : 0.0;
    precise float qf = roundEven(h * id);
    q_s[t] = int(clamp(qf, -127.0, 127.0));
    barrier();
    uint o = blk * 80u; /* 320 bytes = 80 words */
    if (t < 64u)
        X.v[o + t] = uint(q_s[4u * t] & 0xff) | (uint(q_s[4u * t + 1u] & 0xff) << 8) |
                     (uint(q_s[4u * t + 2u] & 0xff) << 16) |
                     (uint(q_s[4u * t + 3u] & 0xff) << 24);
    else if (t < 72u) {
        /* two pair sums per word: the sums of 16 quants */
        uint k = t - 64u;
        int s0 = 0, s1 = 0;
        for (uint i = 0; i < 16u; i++) {
            s0 += q_s[32u * k + i];
            s1 += q_s[32u * k + 16u + i];
        }
        X.v[o + t] = uint(s0 & 0xffff) | (uint(s1 & 0xffff) << 16);
    } else if (t == 72u)
        X.v[o + t] = floatBitsToUint(d);
}