Skip to content

armgit / janas

Public Imported from GitHub
0

37 tok/s from Qwen3-Next-80B (48 GB) on a 32 GB laptop, 24 without drafts vs 2.8 for llama.cpp; reading 62 vs 16 tok/s. Faster than llama.cpp on all 13 models compared in Q4_K_M. LLMs bigger than your RAM, pure C, CPU or GPU.

5.0 KB
Raw Blame
/* SPDX-License-Identifier: GPL-3.0-or-later
   Copyright (C) 2026 Maurizio Cammalleri */
#version 450
// The Gated DeltaNet recurrence of a block, as deltanet.c's gdn_step_h does
// it token after token: per value head h and row r of its state M (ds = 128
// numbers, kept at sixteen bits between tokens), with the token's q and k
// (key head g), v and the head's beta and decay -
//   x = M[r] decay, sk = x . k, d = (v[r] - sk) beta, y = x + d k,
//   out[r] = (y . q) scale, M[r] = y at sixteen bits.
// The same operations, not the same bits: the dot products are summed in
// another order (clusters of eight lanes) and packHalf2x16 rounds the state
// as the GPU does, so the results agree with the CPU's to rounding.
// The row stays in registers for the whole block: eight threads a row,
// sixteen numbers each (their q and k read four at a time), sixteen rows a
// workgroup. The state is read from the half the position buffer's word 1
// names and written to the one its word 2 does.
// A call that starts before the end of the last one first replays the a
// tokens of it that were kept (the position buffer's word 3): their rows of
// the convolution come first, their beta and decay from the log (lb, ld:
// the layer's, nv a row), their outputs are not wanted, and the state after
// them goes back into the half it was read from (the base: a call that
// replays starts from it); then the call's n tokens, their beta and decay
// from bd (gdn_ba.comp or gdn_act.comp: per token the nv betas, then the nv
// decays).
#extension GL_EXT_buffer_reference : require
#extension GL_EXT_buffer_reference_uvec2 : require
#extension GL_EXT_control_flow_attributes : require
#extension GL_KHR_shader_subgroup_basic : require
#extension GL_KHR_shader_subgroup_clustered : require

layout(local_size_x = 128) in;

layout(buffer_reference, std430, buffer_reference_align = 4) readonly buffer R { float v[]; };
layout(buffer_reference, std430, buffer_reference_align = 16) readonly buffer R4 { vec4 v[]; };
layout(buffer_reference, std430, buffer_reference_align = 4) writeonly buffer W { float v[]; };
layout(buffer_reference, std430, buffer_reference_align = 4) buffer U { uint v[]; };
layout(buffer_reference, std430, buffer_reference_align = 4) readonly buffer RP { uint v[]; };

// co: a + n rows of cc (q heads, k heads at kd, v at 2 kd), on 16 bytes;
// att: n rows of nv ds; state: two halves of hs sixteen-bit numbers, this
// layer's nv ds ds at so; lb, ld: the log's betas and decays, nv a row; bd:
// n rows of 2 nv
layout(push_constant) uniform P {
    uvec2 co, att, state, pos, lb, ld, bd;
    uint cc, nv, nkh, n, hs, so, vmap_mod;
    float scale;
} p;

void main()
{
    uint t = gl_LocalInvocationID.x, lane = t % 8u, rl = t / 8u;
    uint h = gl_WorkGroupID.x / 8u, r = (gl_WorkGroupID.x % 8u) * 16u + rl;
    uint kd = p.nkh * 128u, g = p.vmap_mod != 0u ? h % p.nkh : h / (p.nv / p.nkh);
    R CO = R(p.co);
    R4 CO4 = R4(p.co);
    W O = W(p.att);
    U S = U(p.state);
    RP PS = RP(p.pos);
    /* this thread's sixteen numbers of the row: e0 .. e0 + 15 */
    uint e0 = lane * 16u;
    uint row = p.so + (h * 128u + r) * 128u + e0; /* sixteen-bit numbers */
    uint src = (PS.v[1] * p.hs + row) / 2u, dst = (PS.v[2] * p.hs + row) / 2u;
    float x[16];
    [[unroll]] for (uint i = 0; i < 8u; i++) {
        vec2 f = unpackHalf2x16(S.v[src + i]);
        x[2u * i] = f.x;
        x[2u * i + 1u] = f.y;
    }
    uint ar = PS.v[3]; /* the tokens replayed */
    for (uint j = 0; j < ar + p.n; j++) {
        float beta, dec;
        if (j < ar) {
            beta = R(p.lb).v[j * p.nv + h];
            dec = R(p.ld).v[j * p.nv + h];
        } else {
            uint jn = (j - ar) * 2u * p.nv + h;
            beta = R(p.bd).v[jn];
            dec = R(p.bd).v[jn + p.nv];
        }
        /* the token's q and k, sixteen numbers each, a vec4 at a time */
        uint qo = (j * p.cc + g * 128u + e0) / 4u, ko = qo + kd / 4u;
        vec4 kv[4], qv[4];
        [[unroll]] for (uint m = 0; m < 4u; m++) {
            kv[m] = CO4.v[ko + m];
            qv[m] = CO4.v[qo + m];
        }
        float sk = 0.0;
        [[unroll]] for (uint i = 0; i < 16u; i++) {
            x[i] *= dec;
            sk = fma(x[i], kv[i / 4u][i % 4u], sk);
        }
        sk = subgroupClusteredAdd(sk, 8u);
        float d = (CO.v[j * p.cc + 2u * kd + h * 128u + r] - sk) * beta;
        float sq = 0.0;
        [[unroll]] for (uint i = 0; i < 16u; i++) {
            float y = fma(d, kv[i / 4u][i % 4u], x[i]);
            sq = fma(y, qv[i / 4u][i % 4u], sq);
            x[i] = unpackHalf2x16(packHalf2x16(vec2(y, 0.0))).x;
        }
        sq = subgroupClusteredAdd(sq, 8u);
        if (lane == 0u && j >= ar)
            O.v[(j - ar) * p.nv * 128u + h * 128u + r] = sq * p.scale;
        if (j + 1u == ar) /* the base, past the replayed tokens: in place */
            [[unroll]] for (uint i = 0; i < 8u; i++)
                S.v[src + i] = packHalf2x16(vec2(x[2u * i], x[2u * i + 1u]));
    }
    [[unroll]] for (uint i = 0; i < 8u; i++)
        S.v[dst + i] = packHalf2x16(vec2(x[2u * i], x[2u * i + 1u]));
}