Skip to content

armgit / janas

Public Imported from GitHub
0

37 tok/s from Qwen3-Next-80B (48 GB) on a 32 GB laptop, 24 without drafts vs 2.8 for llama.cpp; reading 62 vs 16 tok/s. Faster than llama.cpp on all 13 models compared in Q4_K_M. LLMs bigger than your RAM, pure C, CPU or GPU.

2.4 KB
Raw Blame
/* SPDX-License-Identifier: GPL-3.0-or-later
   Copyright (C) 2026 Maurizio Cammalleri */
#version 450
// qwen4exp's indexer (model_qsa.c janas_m_qsa), first step for token j of a
// block (workgroup y) at position p = pos + j (pos the position buffer's
// word 0): workgroup x = h < nh normalizes query head h (rms norm with the
// weights qn) and turns its first n_rot numbers for p (NeoX halves, the
// cos and sin of row j of rope), in place; x = nh writes the token's raw
// key at half precision to row p of raw (d numbers a row). The sums in
// the GPU's order.
#extension GL_EXT_buffer_reference : require
#extension GL_EXT_buffer_reference_uvec2 : require

layout(local_size_x = 256) in;

layout(buffer_reference, std430, buffer_reference_align = 4) buffer F { float v[]; };
layout(buffer_reference, std430, buffer_reference_align = 4) readonly buffer R { float v[]; };
layout(buffer_reference, std430, buffer_reference_align = 4) writeonly buffer U { uint v[]; };
layout(buffer_reference, std430, buffer_reference_align = 4) readonly buffer RP { uint v[]; };

layout(push_constant) uniform P {
    uvec2 q, k, qn, rope, raw, pos;
    uint nh, d, n_rot, qs; /* qs: floats from a token's heads to the next's */
    float eps;
} p;

shared float sv[256];
shared float ss[256];

void main()
{
    uint t = gl_LocalInvocationID.x, h = gl_WorkGroupID.x, j = gl_WorkGroupID.y;
    uint d = p.d;
    if (h == p.nh) {
        /* the raw key, two halves a word */
        uint row = RP(p.pos).v[0] + j;
        R K = R(p.k);
        if (t < d / 2u)
            U(p.raw).v[row * (d / 2u) + t] =
                packHalf2x16(vec2(K.v[j * d + 2u * t], K.v[j * d + 2u * t + 1u]));
        return;
    }
    F Q = F(p.q);
    uint at = j * p.qs + h * d;
    float x = t < d ? Q.v[at + t] : 0.0;
    ss[t] = x * x;
    barrier();
    for (uint s = 128u; s > 0u; s >>= 1) {
        if (t < s)
            ss[t] += ss[t + s];
        barrier();
    }
    float scale = 1.0 / sqrt(ss[0] / float(d) + p.eps);
    if (t < d)
        sv[t] = x * scale * R(p.qn).v[t];
    barrier();
    uint half_ = p.n_rot / 2u;
    if (t < d) {
        precise float v = sv[t]; /* rounded as the CPU's */
        if (t < p.n_rot) {
            R RO = R(p.rope);
            uint i = t % half_;
            float c = RO.v[j * p.n_rot + i], s = RO.v[j * p.n_rot + half_ + i];
            v = t < half_ ? sv[t] * c - sv[t + half_] * s : sv[t - half_] * s + sv[t] * c;
        }
        Q.v[at + t] = v;
    }
}