Skip to content

armgit / janas

Public Imported from GitHub
0

37 tok/s from Qwen3-Next-80B (48 GB) on a 32 GB laptop, 24 without drafts vs 2.8 for llama.cpp; reading 62 vs 16 tok/s. Faster than llama.cpp on all 13 models compared in Q4_K_M. LLMs bigger than your RAM, pure C, CPU or GPU.

2.5 KB
Raw Blame
/* SPDX-License-Identifier: GPL-3.0-or-later
   Copyright (C) 2026 Maurizio Cammalleri */
#version 450
// After gdn_conv.comp: the layer's convolution state, the last d - 1 rows
// of the inputs - the state the call starts from (the half the position
// buffer's word 1 names), then the a logged rows it replays (word 3), then
// its n rows of mixed - for the next call: after the a rows into the half
// it started from (the base, in place: a call that replays starts from the
// base), after all of them into the half word 2 names; any n, as short as
// a draft's. Then the call's rows of mixed into the log, from row word 4 on
// (none if it is all ones), the old rows read first. Apart from
// gdn_conv.comp, which reads the state and the log before. A thread a
// channel.
#extension GL_EXT_buffer_reference : require
#extension GL_EXT_buffer_reference_uvec2 : require

layout(local_size_x = 256) in;

layout(buffer_reference, std430, buffer_reference_align = 4) readonly buffer R { float v[]; };
layout(buffer_reference, std430, buffer_reference_align = 4) buffer S { float v[]; };
layout(buffer_reference, std430, buffer_reference_align = 4) readonly buffer RP { uint v[]; };

layout(push_constant) uniform P { uvec2 mixed, state, pos, log_; uint cc, d, hs, so, n; } p;

/* Row r of the call's inputs, the state's d - 1 rows first: the state,
   the log's a rows, the block's n. */
float input_row(uint r, uint c, uint a, uint base)
{
    uint d1 = p.d - 1u;
    if (r < d1)
        return S(p.state).v[base + r * p.cc + c];
    if (r < d1 + a) /* the log is written below: read as writable too */
        return S(p.log_).v[(r - d1) * p.cc + c];
    return R(p.mixed).v[(r - d1 - a) * p.cc + c];
}

void main()
{
    uint c = gl_WorkGroupID.x * 256u + gl_LocalInvocationID.x;
    if (c >= p.cc)
        return;
    S St = S(p.state);
    RP PS = RP(p.pos);
    uint a = PS.v[3], lrow = PS.v[4];
    uint src = PS.v[1] * p.hs + p.so, dst = PS.v[2] * p.hs + p.so;
    /* both states read before either is written: the halves may be one */
    float nb[7], nd[7];
    for (uint k = 0; k + 1u < p.d; k++) {
        nb[k] = input_row(a + k, c, a, src);
        nd[k] = input_row(a + p.n + k, c, a, src);
    }
    if (a > 0u)
        for (uint k = 0; k + 1u < p.d; k++)
            St.v[src + k * p.cc + c] = nb[k];
    for (uint k = 0; k + 1u < p.d; k++)
        St.v[dst + k * p.cc + c] = nd[k];
    if (lrow != 0xffffffffu) {
        S L = S(p.log_);
        for (uint t = 0; t < p.n; t++)
            L.v[(lrow + t) * p.cc + c] = R(p.mixed).v[t * p.cc + c];
    }
}