Skip to content

armgit / janas

Public Imported from GitHub
0

37 tok/s from Qwen3-Next-80B (48 GB) on a 32 GB laptop, 24 without drafts vs 2.8 for llama.cpp; reading 62 vs 16 tok/s. Faster than llama.cpp on all 13 models compared in Q4_K_M. LLMs bigger than your RAM, pure C, CPU or GPU.

4.9 KB
Raw Blame
/* SPDX-License-Identifier: GPL-3.0-or-later
   Copyright (C) 2026 Maurizio Cammalleri */
#version 450
// A Gated DeltaNet layer's last step before its output product, as
// model_rec.c's gate_worker does: each head's raw output (ds = 128 numbers)
// normalized by its RMS and the weights w (ds), times SiLU(z), then
// quantized to Q8_K as norm_quant.comp does (q256, and q32 if set). A
// workgroup per block of 256 (two heads) and token (y), a number a thread.
// The first workgroup of each token also logs the token's beta and decay of
// each head as gdn_rec.comp used them (bd: 2 nv a token) into lb and ld
// (the layer's log, nv a row) at row (the position buffer's word 4) + y,
// where that word is not all ones.
#extension GL_EXT_buffer_reference : require
#extension GL_KHR_shader_subgroup_arithmetic : require
#extension GL_KHR_shader_subgroup_basic : require
#extension GL_KHR_shader_subgroup_clustered : require
#extension GL_EXT_buffer_reference_uvec2 : require

layout(local_size_x = 256) in;

layout(buffer_reference, std430, buffer_reference_align = 4) readonly buffer F { float v[]; };
layout(buffer_reference, std430, buffer_reference_align = 4) writeonly buffer U { uint v[]; };
layout(buffer_reference, std430, buffer_reference_align = 4) writeonly buffer W { float v[]; };
layout(buffer_reference, std430, buffer_reference_align = 4) readonly buffer RP { uint v[]; };

// att, z: n rows of d floats; w: ds floats
// sig: sigmoid(z) in place of SiLU(z) (qwen4exp)
layout(push_constant) uniform P { uvec2 att, z, w, q256, q32, bd, lb, ld, pos; uint d, nv; float eps; uint sig; } p;

shared float sq_[256];
shared float hs_[2];

shared float m8[32];        /* the block's maxima, eight values each */
shared int s8a[32], s8b[32]; /* and its quants' sums, eight each */

bool set(uvec2 a) { return a.x != 0u || a.y != 0u; }

float recip_det(float d)
{
    precise float y = uintBitsToFloat(0x7EF311C3u - floatBitsToUint(d));
    for (int i = 0; i < 3; i++)
        y = fma(y, fma(-d, y, 1.0), y);
    return y;
}

int q8(float v, float id)
{
    precise float q = roundEven(v * id);
    return int(clamp(q, -127.0, 127.0));
}

void main()
{
    uint t = gl_LocalInvocationID.x, b = gl_WorkGroupID.x, j = gl_WorkGroupID.y;
    uint e = 256u * b + t, o = j * p.d + e;
    uint lrow = RP(p.pos).v[4];
    if (b == 0u && t < p.nv && lrow != 0xffffffffu) {
        F BD = F(p.bd);
        W(p.lb).v[(lrow + j) * p.nv + t] = BD.v[j * 2u * p.nv + t];
        W(p.ld).v[(lrow + j) * p.nv + t] = BD.v[j * 2u * p.nv + p.nv + t];
    }
    U Q = U(p.q256), Q32 = U(p.q32);
    float a = F(p.att).v[o];
    sq_[t] = a * a;
    barrier();
    if (t < 2u) { /* each head's sum of squares, in order */
        float s = 0.0;
        for (uint i = 0; i < 128u; i++)
            s += sq_[128u * t + i];
        hs_[t] = s;
    }
    barrier();
    precise float sc = 1.0 / sqrt(hs_[t / 128u] / 128.0 + p.eps);
    precise float vs = a * sc;
    float z = F(p.z).v[o];
    float v = vs * F(p.w).v[e % 128u] * ((p.sig != 0u ? 1.0 : z) / (1.0 + exp(-z)));
    /* the maxima of the groups of 32 and of the block */
    float mc = subgroupClusteredMax(abs(v), 8u);
    if (t % 8u == 0u)
        m8[t / 8u] = mc;
    barrier();
    uint l = t / 32u;
    float m32 = max(max(m8[4u * l], m8[4u * l + 1u]), max(m8[4u * l + 2u], m8[4u * l + 3u]));
    float m256 = 0.0;
    for (uint k = 0; k < 32u; k++)
        m256 = max(m256, m8[k]);
    precise float dd = m256 * 0.007874015718698502;
    float id = dd > 0.0 ? recip_det(dd) : 0.0;
    precise float d32 = m32 * 0.007874015718698502;
    float id32 = d32 > 0.0 ? recip_det(d32) : 0.0;
    int qa = q8(v, id), qb = q8(v, id32);
    /* four quants a word, from four neighbouring threads */
    uint wa = subgroupClusteredOr((uint(qa) & 0xffu) << (8u * (t % 4u)), 4u);
    uint wb = subgroupClusteredOr((uint(qb) & 0xffu) << (8u * (t % 4u)), 4u);
    uint base = (j * gl_NumWorkGroups.x + b) * 80u; /* 320 bytes a block */
    if (t % 4u == 0u) {
        Q.v[base + t / 4u] = wa;
        if (set(p.q32))
            Q32.v[base + t / 4u] = wb;
    }
    /* the sums of 16 quants: two of eight each */
    int ca = subgroupClusteredAdd(qa, 8u), cb = subgroupClusteredAdd(qb, 8u);
    if (t % 8u == 0u) {
        s8a[t / 8u] = ca;
        s8b[t / 8u] = cb;
    }
    barrier();
    if (t < 8u) {
        int sa0 = s8a[4u * t] + s8a[4u * t + 1u], sa1 = s8a[4u * t + 2u] + s8a[4u * t + 3u];
        Q.v[base + 64u + t] = (uint(sa0) & 0xffffu) | (uint(sa1) << 16);
        if (set(p.q32)) {
            int sb0 = s8b[4u * t] + s8b[4u * t + 1u], sb1 = s8b[4u * t + 2u] + s8b[4u * t + 3u];
            Q32.v[base + 64u + t] = (uint(sb0) & 0xffffu) | (uint(sb1) << 16);
            precise float dt =
                max(max(m8[4u * t], m8[4u * t + 1u]), max(m8[4u * t + 2u], m8[4u * t + 3u])) *
                0.007874015718698502;
            Q32.v[base + 72u + t] = floatBitsToUint(dt);
        }
    }
    if (t == 0u)
        Q.v[base + 72u] = floatBitsToUint(dd);
}