Skip to content

Run frontier MoE models on hardware you already own — pure C, zero deps, experts streamed from disk. Tiny engine, immense model. 🐦

C
3.2 KB
Raw Blame
#version 450
// DeepSeek V4's routed expert between its activation (act 2 of qmatmul_gate_up.comp
// and expert_act.comp) and down, in place over one expert's hidden rows:
//   hid[r][i] = e4m3(bf16(hid[r][i] * w[r]))
// with w[r] the row's route weight, bf16 rounding to nearest even, and e4m3 the
// quantize-dequantize the engine's fp4 kernel applies to down's input on the CPU
// (coli_fp8_activation_qdq_ref, blocks of 128): per block a power-of-two scale
// 2^e with e the smallest integer for which max(amax, 1e-4) <= 448 * 2^e, each value
// divided by it, clamped to [-448, 448], rounded to E4M3 (nearest even, subnormals
// in steps of 2^-9) and multiplied back. Every step is exact in f32 or rounds once
// as the CPU does, so a row equals the CPU's whenever its activation does.
// One workgroup per (block, row); a block past the end of a row is partial.
layout(local_size_x = 128) in;

layout(std430, binding = 0) buffer H { float hid[]; };
layout(std430, binding = 1) readonly buffer W { float rw[]; };

layout(push_constant) uniform P {
    int rows;
    int n;          // elements per row: moe_inter
} p;

shared float amax[128];

float bf16r(float v) {
    uint b = floatBitsToUint(v);
    if ((b & 0x7f800000u) != 0x7f800000u) b += 0x7fffu + ((b >> 16) & 1u);
    return uintBitsToFloat(b & 0xffff0000u);
}
float pow2(int k) { return uintBitsToFloat(uint(k + 127) << 23); }   // 2^k, -126 <= k <= 127

// |x| <= 448: the E4M3 value nearest to x (ties to even), as coli_e4m3fn_encode
// followed by coli_e4m3fn_decode
float e4m3r(float x) {
    float a = abs(x), r;
    if (a == 0.0) return x;
    if (a >= 448.0) r = 448.0;
    else if (a < 0.015625) {             // below 2^-6: subnormal codes, steps of 2^-9
        float s = a * 512.0, f = floor(s), fr = s - f;
        if (fr > 0.5 || (fr == 0.5 && (int(f) & 1) == 1)) f += 1.0;
        r = f * 0.001953125;
    } else {                             // 1.mmm x 2^E, three mantissa bits
        uint b = floatBitsToUint(a);
        int E = int((b >> 23) & 0xffu) - 127;
        uint sig = 0x800000u | (b & 0x7fffffu), q = sig >> 20, rem = sig & 0xfffffu;
        if (rem > 0x80000u || (rem == 0x80000u && (q & 1u) == 1u)) q++;
        r = float(q) * pow2(E - 3);
    }
    return x < 0.0 ? -r : r;
}

void main() {
    int t = int(gl_LocalInvocationID.x), r = int(gl_WorkGroupID.y);
    int i = int(gl_WorkGroupID.x) * 128 + t;
    bool live = r < p.rows && i < p.n;
    uint at = uint(r) * uint(p.n) + uint(max(i, 0));
    float v = live ? bf16r(hid[at] * rw[r]) : 0.0;
    amax[t] = abs(v);
    barrier();
    for (int s = 64; s > 0; s >>= 1) {
        if (t < s) amax[t] = max(amax[t], amax[t + s]);
        barrier();
    }
    // m = (1 + man / 2^23) * 2^E: m <= 448 * 2^e, 448 = 1.75 * 2^8, holds from
    // e = E - 8 when the mantissa is at most 1.75, else from e = E - 7. (The CPU
    // divides by 448 and takes the ceiling of the log; its quotient is correctly
    // rounded, so the two agree, a power of two included.)
    float m = max(amax[0], 1e-4);
    uint mb = floatBitsToUint(m);
    int E = int((mb >> 23) & 0xffu) - 127;
    int e = (mb & 0x7fffffu) <= 0x600000u ? E - 8 : E - 7;
    e = clamp(e, -126, 126);
    float x = clamp(v * pow2(-e), -448.0, 448.0);
    if (live) hid[at] = e4m3r(x) * pow2(e);
}