Skip to content

Run frontier MoE models on hardware you already own — pure C, zero deps, experts streamed from disk. Tiny engine, immense model. 🐦

C
9.8 KB
Raw Blame
#version 450
// The dense chain's attention core on the matrix units (vk_chain.c, VKC_P_ATTN_FLASH):
// chain_attn.comp's causal case (no selection list, window, ring, sink or second V
// width, head-major cache) for a prompt chunk, KHR cooperative matrix f16 x f16 -> f32.
// Same bindings and push constants as chain_attn.comp, so vkc_attn_w picks either.
// After llama.cpp's flash_attn_cm1.comp (MIT): 16 query rows, 64 keys a step, 4
// subgroups of 64, the softmax on a shared copy of the scores. For Qwen3.6 on RADV:
//  * GQA folded: the 16 rows are 16/GQ tokens x the GQ query heads of ONE kv head, so
//    every K/V block is staged once for all of them;
//  * two passes instead of an online softmax: pass 1 finds each row's max m and sum l,
//    pass 2 recomputes the scores and adds exp(s - m) * V straight into the
//    accumulators (no rescaling, O never leaves the registers), / l at the end;
//  * K/V arrive f32 and are converted to f16 while staged, in 16x16 tiles stored
//    contiguously (strided cooperative-matrix loads measured slow on RADV);
//  * causality inline: key t is valid for row r iff t <= pos_base + token(r).
// Q is scaled before its f16 rounding; P = exp(s - m) is in (0, 1]. Not bit-identical
// to chain_attn.comp (f16 operands); the CPU's order differs from both anyway.
// Grid: x = ceil(S / (16/GQ)) token groups, y = kv heads. HD and GQ are specialization
// constants (HD a multiple of 64 up to 256, GQ dividing 16).
#extension GL_KHR_cooperative_matrix : require
#extension GL_KHR_memory_scope_semantics : require
#extension GL_EXT_shader_explicit_arithmetic_types_float16 : require
#extension GL_EXT_control_flow_attributes : enable
#extension GL_KHR_shader_subgroup_basic : require
#extension GL_KHR_shader_subgroup_clustered : require
layout(local_size_x = 256) in;
layout(constant_id = 0) const uint HD = 256;   // head dim, multiple of 64
layout(constant_id = 1) const uint GQ = 8;     // query heads per kv head, divides 16
#define TPW (16u / GQ)                         // tokens per workgroup
#define BC 64u                                 // keys per block
#define NDT (HD / 16u)                         // 16-wide head-dim tiles
#define NDS (NDT / 4u)                         // head-dim tiles of O per subgroup

layout(std430, binding = 0) readonly buffer QB { float q[]; };
layout(std430, binding = 1) readonly buffer KB { float kc[]; };   // K[k_off + (kvh*cap + t)*hd + d]
layout(std430, binding = 2) readonly buffer VB { float vc[]; };   // V likewise from v_off
layout(std430, binding = 3) buffer OB { float o[]; };
layout(std430, binding = 4) readonly buffer GB { float gt[]; };
layout(push_constant) uniform P {   // chain_attn.comp's; the fields after v_off are zero here
    int S, H, KVH, hd, pos_base, cap;
    int q_off, q_row, q_seg;
    int g_off, g_row, g_seg, has_gate;
    int o_off, o_row;
    int sel_off, sel_row;
    float scale;
    int k_off, v_off;
    int win, ring, vd, kv_pm, sink, sink_off;
} p;

// K or V block (or Q before the loop), f16, 16x16 tiles of 256 values each:
// tile (a, b) at (a * NDT + b) * 256, element (row, col) at row * 16 + col.
// K: a = key tile, b = dim tile, row = key, col = dim -> loaded ColumnMajor as B(dim, key)
// V: same layout                                     -> loaded RowMajor as B(key, dim)
shared uvec4 skv[BC * HD / 8u];
shared float ss[4u * 256u];            // scores, tile per subgroup (row, key); output staging at the end
shared float16_t sp[4u * 256u];        // P as 4 A tiles (row, key)
shared float sinvl[16];                // 1 / l per row

uint pk(float a, float b) { return packFloat2x16(f16vec2(float16_t(a), float16_t(b))); }

// stage keys k0 .. k0+63 of kv head g from f32 memory into skv (zero past nt)
void stage(uint g, uint k0, uint nt, uint tid, bool isv) {
    for (uint e = tid; e < BC * HD / 8u; e += 256u) {
        uint kl = e / (HD / 8u), d8 = e % (HD / 8u);   // consecutive threads: consecutive dims of one key
        uint key = k0 + kl, d = d8 * 8u;
        vec4 a = vec4(0.0), b = vec4(0.0);
        if (key < nt) {
            uint base = (g * uint(p.cap) + key) * HD + d;
            if (isv) { base += uint(p.v_off); a = vec4(vc[base], vc[base+1u], vc[base+2u], vc[base+3u]); b = vec4(vc[base+4u], vc[base+5u], vc[base+6u], vc[base+7u]); }
            else     { base += uint(p.k_off); a = vec4(kc[base], kc[base+1u], kc[base+2u], kc[base+3u]); b = vec4(kc[base+4u], kc[base+5u], kc[base+6u], kc[base+7u]); }
        }
        skv[((kl >> 4u) * NDT + (d8 >> 1u)) * 32u + (kl & 15u) * 2u + (d8 & 1u)] = uvec4(pk(a.x, a.y), pk(a.z, a.w), pk(b.x, b.y), pk(b.z, b.w));
    }
}

void main() {
    uint tid = gl_LocalInvocationID.x, sgi = gl_SubgroupID, lane = gl_SubgroupInvocationID;
    uint S = uint(p.S), H = uint(p.H), g = gl_WorkGroupID.y;
    uint t0 = gl_WorkGroupID.x * TPW;
    uint nt = uint(p.pos_base) + S;
    uint tlast = min(t0 + TPW, S) - 1u;
    uint nkb = (uint(p.pos_base) + tlast) / BC + 1u;          // key blocks this workgroup needs

    // softmax thread layout: row = tid / 16, 16 lanes per row, keys cl + 16 i
    uint row = tid >> 4u, cl = tid & 15u;
    uint rtok = t0 + row / GQ;
    uint qpos = uint(p.pos_base) + min(rtok, S - 1u);

    // Q (scaled) -> f16 A tiles in registers
    coopmat<float16_t, gl_ScopeSubgroup, 16, 16, gl_MatrixUseA> A[NDT];
    for (uint e = tid; e < 16u * HD / 8u; e += 256u) {
        uint r = e / (HD / 8u), d8 = e % (HD / 8u), tk = t0 + r / GQ, h = g * GQ + r % GQ;
        vec4 a = vec4(0.0), b = vec4(0.0);
        if (tk < S) {
            uint base = uint(p.q_off) + tk * uint(p.q_row) + h * uint(p.q_seg) + d8 * 8u;
            a = vec4(q[base], q[base+1u], q[base+2u], q[base+3u]) * p.scale;
            b = vec4(q[base+4u], q[base+5u], q[base+6u], q[base+7u]) * p.scale;
        }
        skv[(d8 >> 1u) * 32u + r * 2u + (d8 & 1u)] = uvec4(pk(a.x, a.y), pk(a.z, a.w), pk(b.x, b.y), pk(b.z, b.w));
    }
    barrier();
    [[unroll]] for (uint j = 0u; j < NDT; j++) coopMatLoad(A[j], skv, j * 32u, 2u, gl_CooperativeMatrixLayoutRowMajor);
    barrier();

    coopmat<float16_t, gl_ScopeSubgroup, 16, 16, gl_MatrixUseB> B;
    coopmat<float, gl_ScopeSubgroup, 16, 16, gl_MatrixUseAccumulator> Sc;
    float m = -1e30, l = 0.0;

    // pass 1: row max and sum
    for (uint kb = 0u; kb < nkb; kb++) {
        stage(g, kb * BC, nt, tid, false);
        barrier();
        Sc = coopmat<float, gl_ScopeSubgroup, 16, 16, gl_MatrixUseAccumulator>(0.0);
        [[unroll]] for (uint j = 0u; j < NDT; j++) {
            coopMatLoad(B, skv, (sgi * NDT + j) * 32u, 2u, gl_CooperativeMatrixLayoutColumnMajor);
            Sc = coopMatMulAdd(A[j], B, Sc);
        }
        coopMatStore(Sc, ss, sgi * 256u, 16u, gl_CooperativeMatrixLayoutRowMajor);
        barrier();
        float sv[4], bm = -1e30;
        [[unroll]] for (uint i = 0u; i < 4u; i++) {
            uint key = kb * BC + i * 16u + cl;
            sv[i] = key <= qpos ? ss[i * 256u + row * 16u + cl] : -1e30;
            bm = max(bm, sv[i]);
        }
        bm = subgroupClusteredMax(bm, 16u);
        float mn = max(m, bm), part = 0.0;
        [[unroll]] for (uint i = 0u; i < 4u; i++) part += sv[i] > -1e29 ? exp(sv[i] - mn) : 0.0;
        l = l * exp(m - mn) + subgroupClusteredAdd(part, 16u);
        m = mn;
        barrier();                                   // skv and ss are rewritten next block
    }
    // P stays exp(s - m) in (0, 1] and O is divided by l at the end: normalised
    // probabilities of a long row fall below f16's normal range (6e-5)
    if (cl == 0u) sinvl[row] = 1.0 / l;

    // pass 2: O = sum_t exp(s - m) * V  (/ l at the end)
    coopmat<float, gl_ScopeSubgroup, 16, 16, gl_MatrixUseAccumulator> Oa[NDS];
    [[unroll]] for (uint u = 0u; u < NDS; u++) Oa[u] = coopmat<float, gl_ScopeSubgroup, 16, 16, gl_MatrixUseAccumulator>(0.0);
    coopmat<float16_t, gl_ScopeSubgroup, 16, 16, gl_MatrixUseA> PA;
    for (uint kb = 0u; kb < nkb; kb++) {
        stage(g, kb * BC, nt, tid, false);
        barrier();
        Sc = coopmat<float, gl_ScopeSubgroup, 16, 16, gl_MatrixUseAccumulator>(0.0);
        [[unroll]] for (uint j = 0u; j < NDT; j++) {
            coopMatLoad(B, skv, (sgi * NDT + j) * 32u, 2u, gl_CooperativeMatrixLayoutColumnMajor);
            Sc = coopMatMulAdd(A[j], B, Sc);
        }
        coopMatStore(Sc, ss, sgi * 256u, 16u, gl_CooperativeMatrixLayoutRowMajor);
        barrier();                                   // scores complete, K block consumed
        [[unroll]] for (uint i = 0u; i < 4u; i++) {
            uint key = kb * BC + i * 16u + cl;
            float s = ss[i * 256u + row * 16u + cl];
            sp[i * 256u + row * 16u + cl] = float16_t(key <= qpos ? exp(s - m) : 0.0);
        }
        stage(g, kb * BC, nt, tid, true);
        barrier();
        [[unroll]] for (uint kt = 0u; kt < 4u; kt++) {
            coopMatLoad(PA, sp, kt * 256u, 16u, gl_CooperativeMatrixLayoutRowMajor);
            [[unroll]] for (uint u = 0u; u < NDS; u++) {
                coopMatLoad(B, skv, (kt * NDT + sgi * NDS + u) * 32u, 2u, gl_CooperativeMatrixLayoutRowMajor);
                Oa[u] = coopMatMulAdd(PA, B, Oa[u]);
            }
        }
        barrier();
    }

    // out: rows map to (token, head), not a uniform stride -> via this subgroup's ss tile
    [[unroll]] for (uint u = 0u; u < NDS; u++) {
        coopMatStore(Oa[u], ss, sgi * 256u, 16u, gl_CooperativeMatrixLayoutRowMajor);
        subgroupMemoryBarrierShared(); subgroupBarrier();
        for (uint e = lane; e < 256u; e += gl_SubgroupSize) {
            uint r = e >> 4u, c = e & 15u, tk = t0 + r / GQ, h = g * GQ + r % GQ;
            if (tk < S) {
                uint d = (sgi * NDS + u) * 16u + c;
                float v = ss[sgi * 256u + e] * sinvl[r];
                if (p.has_gate != 0) v *= 1.0 / (1.0 + exp(-gt[uint(p.g_off) + tk * uint(p.g_row) + h * uint(p.g_seg) + d]));
                o[uint(p.o_off) + tk * uint(p.o_row) + h * HD + d] = v;
            }
        }
        subgroupMemoryBarrierShared(); subgroupBarrier();
    }
}