Skip to content

Run frontier MoE models on hardware you already own — pure C, zero deps, experts streamed from disk. Tiny engine, immense model. 🐦

C
5.3 KB
Raw Blame
#version 450
// Per-head block matmuls of a resident tensor on the dense chain (vk_chain.c,
// VKC_P_HGEMV): the weight halves of multi-head latent attention, where every head
// multiplies its own slice of the activations by its own block of rows. Head h's
// block is the rows h*hstride + hoff + [0, n) of W ([O x I], any resident format).
//
// mode 0, rows: y[s][h][o] = sum_i W[h*hstride + hoff + o][i] * x[s][h][i], o < n,
//   i < I (the value rows of kv_b times the latent context; an absorbed query from a
//   key matrix stored per head as [K x Q]). One subgroup per output row, its lanes
//   split the row. With has_gate, y *= sigmoid(g[g_off + s*g_row + h*n + o]) (an
//   output gate on the values).
// mode 1, transposed: y[s][h][i] = sum_d W[h*hstride + hoff + d][i] * x[s][h][d],
//   d < n, i < I (the absorbed query from kv_b's key rows: q_nope's n values times
//   the head's n rows, into the latent's I columns). One workgroup per (h, s), x's
//   slice staged in shared memory (n <= 1024), a thread per column, d in order.
// x[s][h][.] is at x_off + s*x_row + h*x_seg, y[s][h][.] at y_off + s*y_row + h*y_seg.
//
// Formats (the backend's, qmatmul.comp's layout: row r at r*rowWords words):
//   1 int8 rows (scale per row), 2 int4 rows (nibble - 8, scale per row), 4 int4 in
//   groups of gs, 5 int3-g64 (24 bytes and one scale per 64), 7 MXFP4 (scale per gs),
//   10 f32, 11 bf16 (low half = even column), 12 fp8 e4m3 (scale per gs), 13 int8 in
//   groups of gs.
#extension GL_KHR_shader_subgroup_arithmetic : require
layout(local_size_x = 256) in;

layout(std430, binding = 0) readonly buffer X { float x[]; };
layout(std430, binding = 1) readonly buffer W { uint w[]; };
layout(std430, binding = 2) readonly buffer Sc { float sc[]; };
layout(std430, binding = 3) writeonly buffer Y { float y[]; };
layout(std430, binding = 4) readonly buffer G { float g[]; };

layout(push_constant) uniform P {
    int mode, fmt, I, rowWords, gs;
    int S, H, n, hstride, hoff;
    int x_off, x_row, x_seg, y_off, y_row, y_seg;
    int g_off, g_row, has_gate;
} p;

shared float xs[1024];

float i8(uint word, int lane) { return float(bitfieldExtract(int(word), lane * 8, 8)); }
float i4(uint word, int lane) { return float(bitfieldExtract(word, lane * 4, 4)) - 8.0; }
float mx4(uint word, int lane) {
    uint nb = bitfieldExtract(word, lane * 4, 4);
    const float lut[8] = float[8](0.0, 0.5, 1.0, 1.5, 2.0, 3.0, 4.0, 6.0);
    float v = lut[nb & 7u];
    return (nb & 8u) != 0u ? -v : v;
}
float e4m3(uint word, int lane) {
    uint b = bitfieldExtract(word, lane * 8, 8);
    if ((b & 0x7fu) == 0x7fu) return uintBitsToFloat(0x7fc00000u);
    uint e = (b >> 3) & 0xfu, m = b & 7u;
    float v = e == 0u ? float(m) * 0.001953125 : ldexp(1.0 + float(m) * 0.125, int(e) - 7);
    return (b & 0x80u) != 0u ? -v : v;
}

// W[row][i], dequantized
float wv(int row, int i) {
    uint rb = uint(row) * uint(p.rowWords);
    int f = p.fmt;
    if (f == 10) return uintBitsToFloat(w[rb + uint(i)]);
    if (f == 11) { uint pk = w[rb + uint(i >> 1)]; return uintBitsToFloat((i & 1) != 0 ? (pk & 0xffff0000u) : (pk << 16)); }
    if (f == 1) return i8(w[rb + uint(i >> 2)], i & 3) * sc[row];
    if (f == 2) return i4(w[rb + uint(i >> 3)], i & 7) * sc[row];
    int ng = (p.I + p.gs - 1) / max(p.gs, 1);
    if (f == 4) return i4(w[rb + uint(i >> 3)], i & 7) * sc[row * ng + i / p.gs];
    if (f == 7) return mx4(w[rb + uint(i >> 3)], i & 7) * sc[row * ng + i / p.gs];
    if (f == 12) return e4m3(w[rb + uint(i >> 2)], i & 3) * sc[row * ng + i / p.gs];
    if (f == 13) return i8(w[rb + uint(i >> 2)], i & 3) * sc[row * ng + i / p.gs];
    // 5: int3-g64, 24 bytes per group: a 16-byte plane of 2-bit lows, an 8-byte plane of high bits
    int gi = i >> 6, j = i & 63, ng64 = (p.I + 63) >> 6;
    uint blo = w[rb + uint(gi * 6 + (j >> 4))], bhi = w[rb + uint(gi * 6 + 4 + (j >> 5))];
    uint u = ((blo >> uint((j & 15) * 2)) & 3u) | (((bhi >> uint(j & 31)) & 1u) << 2);
    return float(int(u) - 4) * sc[row * ng64 + gi];
}

void main() {
    int tid = int(gl_LocalInvocationID.x);
    if (p.mode == 0) {
        int s = int(gl_WorkGroupID.y);
        int lane = int(gl_SubgroupInvocationID), sgs = int(gl_SubgroupSize);
        int nsg = 256 / sgs, total = p.H * p.n;
        int stride = int(gl_NumWorkGroups.x) * nsg;
        for (int og = int(gl_WorkGroupID.x) * nsg + int(gl_SubgroupID); og < total; og += stride) {
            int h = og / p.n, o = og - h * p.n;
            int row = h * p.hstride + p.hoff + o;
            int xb = p.x_off + s * p.x_row + h * p.x_seg;
            float a = 0.0;
            for (int i = lane; i < p.I; i += sgs) a += wv(row, i) * x[xb + i];
            float tot = subgroupAdd(a);
            if (lane == 0) {
                if (p.has_gate != 0) tot *= 1.0 / (1.0 + exp(-g[p.g_off + s * p.g_row + h * p.n + o]));
                y[p.y_off + s * p.y_row + h * p.y_seg + o] = tot;
            }
        }
        return;
    }
    int h = int(gl_WorkGroupID.x), s = int(gl_WorkGroupID.y);
    int xb = p.x_off + s * p.x_row + h * p.x_seg;
    for (int d = tid; d < p.n; d += 256) xs[d] = x[xb + d];
    barrier();
    int r0 = h * p.hstride + p.hoff, yb = p.y_off + s * p.y_row + h * p.y_seg;
    for (int i = tid; i < p.I; i += 256) {
        float a = 0.0;
        for (int d = 0; d < p.n; d++) a += xs[d] * wv(r0 + d, i);
        y[yb + i] = a;
    }
}