Skip to content

armgit / janas

Public Imported from GitHub
0

37 tok/s from Qwen3-Next-80B (48 GB) on a 32 GB laptop, 24 without drafts vs 2.8 for llama.cpp; reading 62 vs 16 tok/s. Faster than llama.cpp on all 13 models compared in Q4_K_M. LLMs bigger than your RAM, pure C, CPU or GPU.

4.0 KB
Raw Blame
/* SPDX-License-Identifier: GPL-3.0-or-later
   Copyright (C) 2026 Maurizio Cammalleri */
#version 450
// A mixture's routing on the GPU for a prompt's block (gpu_token.c), as
// model_ffn.c route_tok: per token, the softmax of its ne router logits,
// the k largest (the first of equals), each weight divided by the sum of
// the k (with es set, times its expert's scale: Gemma 4); logit ne, the shared expert's gate (if sh), through a sigmoid, 1
// without one. A workgroup a token, its logits in shared memory, each
// search a reduction over the 64 invocations (one invocation a token read
// some 1300 logits one after the other: 0.5 ms a layer on a single token
// on the laptop's Arc, 24 Oct 2026). NOT bit for bit with the CPU: the
// GPU's exp, the router's sums in another order.
#extension GL_EXT_buffer_reference : require
#extension GL_EXT_buffer_reference_uvec2 : require

layout(local_size_x = 64) in;

layout(buffer_reference, std430, buffer_reference_align = 4) readonly buffer F { float v[]; };
layout(buffer_reference, std430, buffer_reference_align = 4) writeonly buffer FO { float v[]; };
layout(buffer_reference, std430, buffer_reference_align = 4) writeonly buffer UO { uint v[]; };

// r: the logits, rs floats a token; pe, pw: expert and weight of pair
// j k + i; shw: the shared expert's weight of each token
layout(push_constant) uniform P { uvec2 r, pe, pw, shw, es; uint ne, rs, k, n, sh; } p;

const uint MAXK = 16u, MAXE = 512u; // k <= 16, ne <= 512
const uint NONE = 0xffffffffu;

shared float sv[MAXE];
shared float rv[64];
shared uint ri[64];
shared uint taken[MAXE / 32u];
shared uint ids[MAXK];
shared float w[MAXK];

void main()
{
    uint j = gl_WorkGroupID.x, id = gl_LocalInvocationID.x;
    F R = F(p.r);
    uint base = j * p.rs;
    float mx = R.v[base];
    for (uint e = id; e < p.ne; e += 64u) {
        float v = R.v[base + e];
        sv[e] = v;
        mx = max(mx, v);
    }
    if (id < MAXE / 32u)
        taken[id] = 0u;
    rv[id] = mx;
    barrier();
    for (uint s = 32u; s > 0u; s >>= 1) {
        if (id < s)
            rv[id] = max(rv[id], rv[id + s]);
        barrier();
    }
    mx = rv[0];
    barrier();
    float sum = 0.0;
    for (uint e = id; e < p.ne; e += 64u)
        sum += exp(sv[e] - mx);
    rv[id] = sum;
    barrier();
    for (uint s = 32u; s > 0u; s >>= 1) {
        if (id < s)
            rv[id] += rv[id + s];
        barrier();
    }
    sum = rv[0];
    barrier();
    for (uint i = 0u; i < p.k; i++) {
        /* each invocation's largest not taken (the first of equals: e
           grows), then the largest of those, the lower index of equals */
        float bv = 0.0;
        uint be = NONE;
        for (uint e = id; e < p.ne; e += 64u) {
            if ((taken[e / 32u] & (1u << (e % 32u))) != 0u)
                continue;
            if (be == NONE || sv[e] > bv) {
                bv = sv[e];
                be = e;
            }
        }
        rv[id] = bv;
        ri[id] = be;
        barrier();
        for (uint s = 32u; s > 0u; s >>= 1) {
            if (id < s) {
                uint oe = ri[id + s], me = ri[id];
                float ov = rv[id + s], mv = rv[id];
                if (oe != NONE &&
                    (me == NONE || ov > mv || (ov == mv && oe < me))) {
                    rv[id] = ov;
                    ri[id] = oe;
                }
            }
            barrier();
        }
        if (id == 0u) {
            uint best = ri[0];
            taken[best / 32u] |= 1u << (best % 32u);
            ids[i] = best;
            w[i] = exp(rv[0] - mx) / sum;
        }
        barrier();
    }
    if (id != 0u)
        return;
    float wsum = 0.0;
    for (uint i = 0u; i < p.k; i++)
        wsum += w[i];
    UO PE = UO(p.pe);
    FO PW = FO(p.pw);
    for (uint i = 0u; i < p.k; i++) {
        PE.v[j * p.k + i] = ids[i];
        float wi = w[i] / wsum;
        if (p.es.x != 0u || p.es.y != 0u)
            wi *= F(p.es).v[ids[i]];
        PW.v[j * p.k + i] = wi;
    }
    FO SHW = FO(p.shw);
    SHW.v[j] = p.sh != 0u ? 1.0 / (1.0 + exp(-R.v[base + p.ne])) : 1.0;
}