Skip to content

Run frontier MoE models on hardware you already own — pure C, zero deps, experts streamed from disk. Tiny engine, immense model. 🐦

C
3.7 KB
Raw Blame
#version 450
// Kimi K3's attention residuals and its activation on the dense chain (vk_chain.c,
// PM_ARES), two modes.
//
// mode 0, the residual mix (kimi_k3.c's res_mix), for S rows. Row s mixes nb + 1
// vectors of D floats: the block snapshots v_e at b_off + s*b_row + e*D (e < nb) and the
// running prefix v_nb at x_off + s*x_row:
//   score_e = (v_e . w) / sqrt(mean(v_e^2) + eps)            (w at w_off in prm)
//   y[y_off + s*y_row + d] = sum_e softmax(score)_e v_e[d]   (the sum in e order)
// One workgroup per row. The CPU sums the dot and the squares in double; here they are a
// float tree, so the two agree to rounding. The softmax is the CPU's softmax_: the
// maximum, exp(x - max) in order with their sum, then a division. nb <= 15.
//
// mode 1, SiTU-GLU, one thread per element i < n, in the CPU's order (situf_):
//   y[y_off + i] = b1*tanh(g/b1)*sigmoid(g)*b2*tanh(u/b2),  g = x[x_off + i], u = blk[b_off + i]
layout(local_size_x = 256) in;

layout(std430, binding = 0) readonly buffer XB { float xv[]; };   // 0: the prefix; 1: g
layout(std430, binding = 1) readonly buffer BB { float bv[]; };   // 0: the block snapshots; 1: u
layout(std430, binding = 2) readonly buffer WB { float wv[]; };   // 0: prm (the score weight)
layout(std430, binding = 3) writeonly buffer YB { float yv[]; };

layout(push_constant) uniform P { int v[32]; } p;
#define S_      p.v[1]
#define D_      p.v[2]
#define NB      p.v[3]
#define X_OFF   p.v[4]
#define X_ROW   p.v[5]
#define B_OFF   p.v[6]
#define B_ROW   p.v[7]
#define W_OFF   p.v[8]
#define Y_OFF   p.v[9]
#define Y_ROW   p.v[10]
#define EPS     intBitsToFloat(p.v[11])
// mode 1
#define N_      p.v[1]
#define B1      intBitsToFloat(p.v[11])
#define B2      intBitsToFloat(p.v[12])

shared float rms[256];
shared float rdt[256];
shared float sc[16];

void mixrow() {
    int s = int(gl_WorkGroupID.x + gl_WorkGroupID.y * gl_NumWorkGroups.x);
    if (s >= S_) return;                           // uniform across the workgroup
    int tid = int(gl_LocalInvocationID.x);
    for (int e = 0; e <= NB; e++) {
        bool blk = e < NB;
        int base = blk ? B_OFF + s * B_ROW + e * D_ : X_OFF + s * X_ROW;
        float ms = 0.0, dt = 0.0;
        for (int d = tid; d < D_; d += 256) {
            float x = blk ? bv[base + d] : xv[base + d];
            ms += x * x;
            dt += x * wv[W_OFF + d];
        }
        rms[tid] = ms; rdt[tid] = dt;
        barrier();
        for (int off = 128; off > 0; off >>= 1) {
            if (tid < off) { rms[tid] += rms[tid + off]; rdt[tid] += rdt[tid + off]; }
            barrier();
        }
        if (tid == 0) sc[e] = rdt[0] / sqrt(rms[0] / float(D_) + EPS);
        barrier();
    }
    if (tid == 0) {                                // softmax_, in order
        float m = sc[0];
        for (int e = 1; e <= NB; e++) if (sc[e] > m) m = sc[e];
        float sum = 0.0;
        for (int e = 0; e <= NB; e++) { sc[e] = exp(sc[e] - m); sum += sc[e]; }
        for (int e = 0; e <= NB; e++) sc[e] /= sum;
    }
    barrier();
    int xb = X_OFF + s * X_ROW, bb = B_OFF + s * B_ROW, yb = Y_OFF + s * Y_ROW;
    for (int d = tid; d < D_; d += 256) {
        float a = 0.0;
        for (int e = 0; e < NB; e++) a += sc[e] * bv[bb + e * D_ + d];
        a += sc[NB] * xv[xb + d];
        yv[yb + d] = a;
    }
}

// Preserve small SiTU inputs; tanh(x) rounds to x below 2^-12 in f32.
float situ_tanh(float x) { return abs(x) < 0.000244140625 ? x : tanh(x); }

void situ() {
    int i = int(gl_GlobalInvocationID.x + gl_GlobalInvocationID.y * gl_NumWorkGroups.x * 256u);
    if (i >= N_) return;
    float g = xv[X_OFF + i], u = bv[B_OFF + i];
    float sg = 1.0 / (1.0 + exp(-g));
    yv[Y_OFF + i] = B1 * situ_tanh(g / B1) * sg * B2 * situ_tanh(u / B2);
}

void main() {
    if (p.v[0] == 0) mixrow();
    else situ();
}