Skip to content

Run frontier MoE models on hardware you already own — pure C, zero deps, experts streamed from disk. Tiny engine, immense model. 🐦

C
5.6 KB
Raw Blame
#version 450
// Manifold-constrained hyper-connections (mHC) on the dense chain (vk_chain.c,
// VKC_P_MHC), hyper_connections.h's arithmetic for S rows of H residual streams of D
// floats ([S][H*D]), and a clamped SwiGLU. Five modes:
//
// mode 0, the split: one workgroup per row s. r = 1 / sqrt(mean(x^2) + eps) over the
// row's H*D stream values; the mix logits are mix[s][i] * r (mix = the raw products
// hc_fn . x, from a matmul); then, by one thread in the CPU's order:
//   pre[i]  = sigmoid(m[i] * scale[0] + base[i]) + hc_eps
//   post[i] = 2 * sigmoid(m[H+i] * scale[1] + base[H+i])
//   comb    = softmax per row of m[2H + row*H + col] * scale[2] + base[...], + hc_eps,
//             then iters - 1 rounds of Sinkhorn (rows, then columns) after a first
//             column normalization
// into hp[s][0..H) pre, [H..2H) post, [2H..2H+H*H) comb. scale (3) and base (2H+H*H)
// at prm_off in prm.
// mode 1, collapse: y[s][d] = sum_i pre[i] * x[s][i*D + d], the streams in order.
// mode 2, write back: y[s][j*D + d] = sum_i comb[i*H + j] * x[s][i*D + d] + post[j] * b[s][d]
// mode 3, mean: y[s][d] = (sum_i x[s][i*D + d]) / H (the model's last collapse).
// mode 4, clamped SwiGLU: y[i] = silu(min(a[i], lim)) * clamp(b[i], -lim, lim) over n
//   values (GLM-5.3's dense MLPs and shared experts; lim from the config).
// Offsets in floats; a row of x / y is x_row / y_row floats.
layout(local_size_x = 256) in;

layout(std430, binding = 0) readonly buffer XB { float x[]; };    // streams; 4: a
layout(std430, binding = 1) readonly buffer MB { float mx[]; };   // 0: mix; 2: branch; 4: b
layout(std430, binding = 2) buffer HB { float hp[]; };            // pre | post | comb per row
layout(std430, binding = 3) readonly buffer PB { float prm[]; };  // 0: scale, base
layout(std430, binding = 4) writeonly buffer YB { float y[]; };

layout(push_constant) uniform P {
    int mode, S, H, D, iters;
    int x_off, x_row, m_off, m_row, hp_off, hp_row, y_off, y_row, prm_off, n;
    float eps, hc_eps, lim;
} p;

shared float red[256];
shared float ms[96];

float sig(float v) {
    if (v >= 0.0) { float d = exp(-v); return 1.0 / (1.0 + d); }
    float g = exp(v);
    return g / (1.0 + g);
}

void split() {
    int s = int(gl_WorkGroupID.x), tid = int(gl_LocalInvocationID.x);
    int H = p.H, HD = H * p.D, nm = (2 + H) * H;
    int xb = p.x_off + s * p.x_row;
    float a = 0.0;
    for (int i = tid; i < HD; i += 256) { float v = x[xb + i]; a += v * v; }
    red[tid] = a;
    barrier();
    for (int off = 128; off > 0; off >>= 1) {
        if (tid < off) red[tid] += red[tid + off];
        barrier();
    }
    float r = 1.0 / sqrt(red[0] / float(HD) + p.eps);
    for (int i = tid; i < nm; i += 256) ms[i] = mx[p.m_off + s * p.m_row + i] * r;
    barrier();
    if (tid != 0) return;
    int hb = p.hp_off + s * p.hp_row, pb = p.prm_off;
    for (int i = 0; i < H; i++) {
        hp[hb + i] = sig(ms[i] * prm[pb] + prm[pb + 3 + i]) + p.hc_eps;
        hp[hb + H + i] = 2.0 * sig(ms[H + i] * prm[pb + 1] + prm[pb + 3 + H + i]);
    }
    int cb = hb + 2 * H, mo = 2 * H;
    for (int row = 0; row < H; row++) {
        float mxv = -3.0e38;
        for (int col = 0; col < H; col++) {
            float v = ms[mo + row * H + col] * prm[pb + 2] + prm[pb + 3 + mo + row * H + col];
            hp[cb + row * H + col] = v;
            if (v > mxv) mxv = v;
        }
        float sum = 0.0;
        for (int col = 0; col < H; col++) { float v = exp(hp[cb + row * H + col] - mxv); hp[cb + row * H + col] = v; sum += v; }
        for (int col = 0; col < H; col++) hp[cb + row * H + col] = hp[cb + row * H + col] / sum + p.hc_eps;
    }
    float sums[8];
    for (int col = 0; col < H; col++) { float sm = 0.0; for (int row = 0; row < H; row++) sm += hp[cb + row * H + col]; sums[col] = sm; }
    for (int row = 0; row < H; row++) for (int col = 0; col < H; col++) hp[cb + row * H + col] /= sums[col] + p.hc_eps;
    for (int it = 1; it < p.iters; it++) {
        for (int row = 0; row < H; row++) { float sm = 0.0; for (int col = 0; col < H; col++) sm += hp[cb + row * H + col]; sums[row] = sm; }
        for (int row = 0; row < H; row++) for (int col = 0; col < H; col++) hp[cb + row * H + col] /= sums[row] + p.hc_eps;
        for (int col = 0; col < H; col++) { float sm = 0.0; for (int row = 0; row < H; row++) sm += hp[cb + row * H + col]; sums[col] = sm; }
        for (int row = 0; row < H; row++) for (int col = 0; col < H; col++) hp[cb + row * H + col] /= sums[col] + p.hc_eps;
    }
}

void main() {
    if (p.mode == 0) { split(); return; }
    int i = int(gl_GlobalInvocationID.x + gl_GlobalInvocationID.y * gl_NumWorkGroups.x * 256u);
    if (p.mode == 4) {
        if (i >= p.n) return;
        float g = x[p.x_off + i], u = mx[p.m_off + i];
        g = g > p.lim ? p.lim : g;
        u = u < -p.lim ? -p.lim : (u > p.lim ? p.lim : u);
        y[p.y_off + i] = (g / (1.0 + exp(-g))) * u;
        return;
    }
    int H = p.H, D = p.D;
    if (p.mode == 2) {
        if (i >= p.S * H * D) return;
        int s = i / (H * D), r = i - s * H * D, j = r / D, d = r - j * D;
        int xb = p.x_off + s * p.x_row, hb = p.hp_off + s * p.hp_row;
        float v = 0.0;
        for (int k = 0; k < H; k++) v += hp[hb + 2 * H + k * H + j] * x[xb + k * D + d];
        v += hp[hb + H + j] * mx[p.m_off + s * p.m_row + d];
        y[p.y_off + s * p.y_row + j * D + d] = v;
        return;
    }
    if (i >= p.S * D) return;
    int s = i / D, d = i - s * D;
    int xb = p.x_off + s * p.x_row, hb = p.hp_off + s * p.hp_row;
    float v = 0.0;
    if (p.mode == 1) { for (int k = 0; k < H; k++) v += hp[hb + k] * x[xb + k * D + d]; }
    else { for (int k = 0; k < H; k++) v += x[xb + k * D + d]; v = v / float(H); }
    y[p.y_off + s * p.y_row + d] = v;
}