Skip to content

armgit / janas

Public Imported from GitHub
0

37 tok/s from Qwen3-Next-80B (48 GB) on a 32 GB laptop, 24 without drafts vs 2.8 for llama.cpp; reading 62 vs 16 tok/s. Faster than llama.cpp on all 13 models compared in Q4_K_M. LLMs bigger than your RAM, pure C, CPU or GPU.

8.5 KB
Raw Blame
/* SPDX-License-Identifier: GPL-3.0-or-later
   Copyright (C) 2026 Maurizio Cammalleri */
#version 450
// A vector of d floats on its way into the products: x + add (the
// residual, if add is set; times os if os is not 0: Gemma 4's output
// scale) stored at xo (if set), then v = rms_norm(x) * w
// (if w is set; else v = x), quantized to Q8_K in the product shaders'
// layouts - q256 (qs, bsums, d: janas_q8k_quantize_det) and q32 (the eight
// blocks of 32 with a scale each, if q32 is set), and v itself at xn (if
// set: a recurrent layer's f32 products, an MTP block's hidden states). With
// det 0 (the norms) the same bits as the CPU's rms_norm and
// janas_q8k_quantize: the sum of squares in two floats (some 48 bits, as
// the CPU's double), its mean rounded once, the square root, the
// reciprocal and amax / 127 correctly rounded (a product, then the
// remainder by fma); det 1, the deterministic quantization
// (janas_q8k_quantize_det: an activation). One workgroup per block
// of 256 values: each one makes the whole vector's sum of squares, the
// same operations in the same order, so the same bits, then normalizes and
// quantizes its own block, a value a thread. x is only read: the residual
// goes to xo, another buffer, as the other workgroups may still be reading
// x. The sum of squares is the GPU's own (float: each subgroup's, then
// theirs in order); the quantization is the deterministic one, with the
// CPU's operations (amax x 1/127 as a product, the reciprocal by Newton
// steps, rounding to even) - maxima and integer sums, exact in any order.
#extension GL_EXT_buffer_reference : require
#extension GL_KHR_shader_subgroup_arithmetic : require
#extension GL_KHR_shader_subgroup_basic : require
#extension GL_KHR_shader_subgroup_clustered : require
#extension GL_EXT_buffer_reference_uvec2 : require

layout(local_size_x = 256) in;

layout(buffer_reference, std430, buffer_reference_align = 4) buffer F { float v[]; };
layout(buffer_reference, std430, buffer_reference_align = 4) writeonly buffer U { uint v[]; };

layout(push_constant) uniform P { uvec2 x, add, w, xo, q256, q32, xn; uint d; float eps, os, ps; uint det; } p;

/* The workgroup's sum: each subgroup's in one operation, then the
   subgroups' in a fixed order. */
shared float sg_part[64];

float wg_sum(float s)
{
    float r = subgroupAdd(s);
    if (subgroupElect())
        sg_part[gl_SubgroupID] = r;
    barrier();
    float tot = sg_part[0];
    for (uint i = 1u; i < gl_NumSubgroups; i++)
        tot += sg_part[i];
    barrier();
    return tot;
}

shared float m8[32];        /* the block's maxima, eight values each */
shared int s8a[32], s8b[32]; /* and its quants' sums, eight each */

bool set(uvec2 a) { return a.x != 0u || a.y != 0u; }

float recip_det(float d)
{
    precise float y = uintBitsToFloat(0x7EF311C3u - floatBitsToUint(d));
    for (int i = 0; i < 3; i++)
        y = fma(y, fma(-d, y, 1.0), y);
    return y;
}

/* a / b correctly rounded (b normal, positive): the product with the
   reciprocal, then the remainder's share */
float div_cr(float a, float b)
{
    precise float r = recip_det(b);
    precise float q = a * r;
    precise float e = fma(-q, b, a);
    return fma(e, r, q);
}

/* sqrt(a) correctly rounded (a normal, positive) */
float sqrt_cr(float a)
{
    precise float y = inversesqrt(a);
    precise float s = a * y;
    precise float h = 0.5 * y;
    precise float r = fma(-s, s, a);
    return fma(r, h, s);
}

/* the CPU's scale of janas_q8k_quantize (amax / 127) and its reciprocal,
   or the deterministic ones */
float q_scale(float m)
{
    if (p.det != 0u)
        return m * 0.007874015718698502;
    return div_cr(m, 127.0);
}

float q_inv(float d)
{
    if (d <= 0.0)
        return 0.0;
    return p.det != 0u ? recip_det(d) : div_cr(1.0, d);
}

/* hi + lo as two floats, exactly */
void two_sum(inout float hi, inout float lo, float b)
{
    precise float s = hi + b;
    precise float bb = s - hi;
    precise float e = (hi - (s - bb)) + (b - bb);
    hi = s;
    lo = lo + e;
}

shared float ss_hi[256], ss_lo[256];

int q8(float v, float id)
{
    precise float q = roundEven(v * id);
    return int(clamp(q, -127.0, 127.0));
}

void main()
{
    /* vector j of a block (workgroup y): d floats a vector, d / 256
       quantized blocks */
    uint t = gl_LocalInvocationID.x, b = gl_WorkGroupID.x, d = p.d;
    uint j = gl_WorkGroupID.y, xo0 = j * d;
    F X = F(p.x), A = F(p.add), Wt = F(p.w), XO = F(p.xo);
    U Q = U(p.q256), Q32 = U(p.q32);
    float scale = 1.0;
    if (set(p.w) && p.det == 0u) {
        /* the squares exactly (product and fma), summed in two floats,
           each thread's then the threads' in order */
        float hi = 0.0, lo = 0.0;
        for (uint i = t; i < d; i += 256u) {
            precise float xv = X.v[xo0 + i];
            if (set(p.add))
                xv = xv + A.v[xo0 + i];
            if (p.os != 0.0)
                xv = xv * p.os;
            precise float sq = xv * xv;
            precise float se = fma(xv, xv, -sq);
            two_sum(hi, lo, sq);
            lo = lo + se;
        }
        ss_hi[t] = hi;
        ss_lo[t] = lo;
        barrier();
        /* the threads' two floats added in a tree */
        for (uint s = 128u; s > 0u; s >>= 1) {
            if (t < s) {
                float h = ss_hi[t], l = ss_lo[t];
                two_sum(h, l, ss_hi[t + s]);
                ss_hi[t] = h;
                ss_lo[t] = l + ss_lo[t + s];
            }
            barrier();
        }
        hi = ss_hi[0];
        lo = ss_lo[0];
        barrier();
        /* the mean rounded once, as (float)(ss / n) */
        precise float fd = float(d);
        precise float q0 = div_cr(hi, fd);
        precise float rem = fma(-q0, fd, hi) + lo;
        precise float mean = q0 + rem / fd;
        precise float sc = div_cr(1.0, sqrt_cr(mean + p.eps));
        scale = sc;
    } else if (set(p.w)) {
        float ss = 0.0;
        for (uint i = t; i < d; i += 256u) {
            precise float xv = X.v[xo0 + i];
            if (set(p.add))
                xv = xv + A.v[xo0 + i];
            if (p.os != 0.0)
                xv = xv * p.os;
            ss += xv * xv;
        }
        ss = wg_sum(ss);
        precise float sc = 1.0 / sqrt(ss / float(d) + p.eps);
        scale = sc;
    }
    /* this thread's value: element 256 b + t */
    uint e = 256u * b + t;
    precise float xv = X.v[xo0 + e];
    if (set(p.add))
        xv = xv + A.v[xo0 + e];
    if (p.os != 0.0)
        xv = xv * p.os;
    if (set(p.xo))
        XO.v[xo0 + e] = xv;
    float v = xv;
    if (set(p.w)) {
        precise float vs = xv * scale;
        if (p.ps != 0.0) /* Gemma 4's router: times 1 / sqrt(d) first */
            vs = vs * p.ps;
        v = vs * Wt.v[e];
    }
    if (set(p.xn))
        F(p.xn).v[xo0 + e] = v;
    /* the maxima of the groups of 32 and of the block */
    float mc = subgroupClusteredMax(abs(v), 8u);
    if (t % 8u == 0u)
        m8[t / 8u] = mc;
    barrier();
    uint l = t / 32u;
    float m32 = max(max(m8[4u * l], m8[4u * l + 1u]), max(m8[4u * l + 2u], m8[4u * l + 3u]));
    float m256 = 0.0;
    for (uint k = 0; k < 32u; k++)
        m256 = max(m256, m8[k]);
    precise float dd = q_scale(m256);
    float id = q_inv(dd);
    precise float d32 = q_scale(m32);
    float id32 = q_inv(d32);
    int qa = q8(v, id), qb = q8(v, id32);
    /* four quants a word, from four neighbouring threads */
    uint wa = subgroupClusteredOr((uint(qa) & 0xffu) << (8u * (t % 4u)), 4u);
    uint wb = subgroupClusteredOr((uint(qb) & 0xffu) << (8u * (t % 4u)), 4u);
    uint base = (j * gl_NumWorkGroups.x + b) * 80u; /* 320 bytes a block */
    if (t % 4u == 0u) {
        Q.v[base + t / 4u] = wa;
        if (set(p.q32))
            Q32.v[base + t / 4u] = wb;
    }
    /* the sums of 16 quants: two of eight each */
    int ca = subgroupClusteredAdd(qa, 8u), cb = subgroupClusteredAdd(qb, 8u);
    if (t % 8u == 0u) {
        s8a[t / 8u] = ca;
        s8b[t / 8u] = cb;
    }
    barrier();
    if (t < 8u) {
        int sa0 = s8a[4u * t] + s8a[4u * t + 1u], sa1 = s8a[4u * t + 2u] + s8a[4u * t + 3u];
        Q.v[base + 64u + t] = (uint(sa0) & 0xffffu) | (uint(sa1) << 16);
        if (set(p.q32)) {
            int sb0 = s8b[4u * t] + s8b[4u * t + 1u], sb1 = s8b[4u * t + 2u] + s8b[4u * t + 3u];
            Q32.v[base + 64u + t] = (uint(sb0) & 0xffffu) | (uint(sb1) << 16);
            precise float dt =
                q_scale(max(max(m8[4u * t], m8[4u * t + 1u]), max(m8[4u * t + 2u], m8[4u * t + 3u])));
            Q32.v[base + 72u + t] = floatBitsToUint(dt);
        }
    }
    if (t == 0u)
        Q.v[base + 72u] = floatBitsToUint(dd);
}