Run frontier MoE models on hardware you already own — pure C, zero deps, experts streamed from disk. Tiny engine, immense model. 🐦
C
vulkan: read the GEMM's activations through one view of binding 0
Courtney Andrew Richardsonaf326667ea0c
diff --git a/c/shaders/qmatmul_gemm.comp b/c/shaders/qmatmul_gemm.compindex bfbdba24..732508d5 100644--- a/c/shaders/qmatmul_gemm.comp+++ b/c/shaders/qmatmul_gemm.comp@@ -28,8 +28,9 @@ layout(constant_id = 4) const int TN = 4; // activation rows per threadlayout(local_size_x_id = 5) in; // (BM/TM) * (BN/TN) threadslayout(constant_id = 6) const bool PF = true; // fetch the next step during the products-layout(std430, binding = 0) readonly buffer X { float x[]; };-layout(std430, binding = 0) readonly buffer X4 { vec4 x4[]; }; // same buffer, quads+// One view of x: the expert batch binds it as a dynamic buffer, and SPIRV-Cross (under+// MoltenVK's argument buffers) gives a dynamic buffer's offset to only one of its views.+layout(std430, binding = 0) readonly buffer X4 { vec4 x4[]; }; // x[S*I], as quadslayout(std430, binding = 1) readonly buffer W { uint w[]; }; // row o at o*rowWordslayout(std430, binding = 1) readonly buffer W2 { uvec2 w2[]; }; // same buffer, pairslayout(std430, binding = 1) readonly buffer W4 { uvec4 w4[]; }; // same buffer, quads@@ -151,6 +152,8 @@ const int WU = (BM * (BK / 8) + NT - 1) / NT; // weight units per thread andconst int XU = (BN * (BK / 4) + NT - 1) / NT; // activation quads per thread and stepuvec4 rwa[WU], rwb[WU]; vec2 rws[WU]; vec4 rx[XU];+float x_at(uint i) { return x4[i >> 2][i & 3u]; } // x[i], one float of its quad+// Fetch step k0 of this tile into registers (zeros past O, S and I).void fetch_step(int tid, int o0, int s0, int k0) {[[unroll]] for (int r = 0; r < WU; r++) {@@ -165,10 +168,10 @@ void fetch_step(int tid, int o0, int s0, int k0) {uint b = uint(s) * uint(p.I) + uint(k);if ((p.I & 3) == 0) v = x4[b >> 2]; // rows of x start on a quadelse {- v.x = x[b];- if (k + 1 < p.I) v.y = x[b + 1u];- if (k + 2 < p.I) v.z = x[b + 2u];- if (k + 3 < p.I) v.w = x[b + 3u];+ v.x = x_at(b);+ if (k + 1 < p.I) v.y = x_at(b + 1u);+ if (k + 2 < p.I) v.z = x_at(b + 2u);+ if (k + 3 < p.I) v.w = x_at(b + 3u);}}rx[r] = v;diff --git a/c/shaders/qmatmul_gemm.spv b/c/shaders/qmatmul_gemm.spvindex 3b32ea28..1691e167 100644Binary files a/c/shaders/qmatmul_gemm.spv and b/c/shaders/qmatmul_gemm.spv differ