Run frontier MoE models on hardware you already own — pure C, zero deps, experts streamed from disk. Tiny engine, immense model. 🐦
C
vulkan chain: the diffusion attention on the matrix units, opt-in
JustVugg10c41230c583
diff --git a/c/Makefile b/c/Makefileindex 83a32933..8b755323 100644--- a/c/Makefile+++ b/c/Makefile@@ -745,7 +745,7 @@ VK_SPV += shaders/chain_enc.spvVK_SPV += shaders/chain_attn_flash.spv# a diffusion step's attention, every query row over every key row (made on first use;# qwenimage_chain.h)-VK_SPV += shaders/chain_attn_full.spv+VK_SPV += shaders/chain_attn_full.spv shaders/chain_attn_coop.spv# Qwen-Image's VAE decoder: a band's 3x3 taps and the DupUp3D shortcut (made on first use)VK_SPV += shaders/chain_vae.spvendifdiff --git a/c/qwenimage_chain.h b/c/qwenimage_chain.hindex 7863174b..8f1e401e 100644--- a/c/qwenimage_chain.h+++ b/c/qwenimage_chain.h@@ -55,6 +55,7 @@ typedef struct {unsigned long long steps;double wait_ms;int resident, streamed; /* blocks the device keeps (0..resident-1), blocks uploaded each step */+ int coop_attn; /* the attention on the matrix units (int8 weights) */int slot[2]; /* the streamed block in each of the two places (-1: none) */unsigned long long ser[2]; /* the frame that last read it */unsigned long long stream_bytes;@@ -108,6 +109,12 @@ static void qic_plan(Dit *d) {if (R > L) R = L;}g_qic.resident = R; g_qic.streamed = L - R; g_qic.slot[0] = g_qic.slot[1] = -1;+ const char *ae = getenv("COLI_VK_QI_ATTN");+ /* opt-in: on a Radeon 780M the f32 tiles beat the matrix units (a 1024x1024 step's+ * attention 17.9 s against 32.5: RDNA3's f16 MMA is twice its f32 rate, and the two+ * passes cost more than that); a card whose matrix units outrun its f32 by more may+ * gain, not measured yet */+ g_qic.coop_attn = ae && !strcmp(ae, "coop") && d->B[0].q.m.fmt == QI_I8;if (R == L) return;for (int l = R; l < L; l++) { /* the per-matrix path leaves them on the CPU */Lin *m[7]; qic_block_mats(&d->B[l], m);@@ -218,15 +225,17 @@ static int qic_attn(const VkcEncAttn *at, int N, int L) {long long b = vkc_attn_slice_budget(), per = (long long)(L + N) * at->H * at->hd;int rr = b > 0 ? (int)(b / per / 16 * 16) : N;if (rr < 16) rr = 16;- /* chain_attn_full where it loads (head dims 32, 64 and 128), the encoders'- * attention over row ranges otherwise */- int full = (at->hd == 32 || at->hd == 64 || at->hd == 128) && vkc_attn_full_ready(), ok = 1;+ /* on the matrix units with COLI_VK_QI_ATTN=coop and int8 weights (their f16+ * operands cost less than the weights' own rounding), else chain_attn_full where it+ * loads (head dims 32, 64 and 128), else the encoders' attention over row ranges */+ int coop = g_qic.coop_attn && vkc_attn_full_coop_ready(at->hd);+ int full = coop || ((at->hd == 32 || at->hd == 64 || at->hd == 128) && vkc_attn_full_ready()), ok = 1;for (int r0 = 0; ok && r0 < N; r0 += rr) {int n = N - r0 < rr ? N - r0 : rr;if (full) {VkcAttnFull f = {n, L + N, at->H, at->hd, at->q_off + r0 * at->kv_row, at->kv_row, at->k_off, at->v_off,at->kv_row, at->o_off + r0 * at->o_row, at->o_row, at->scale};- ok = vkc_attn_full(g_qic.qkv, g_qic.a, &f);+ ok = coop ? vkc_attn_full_coop(g_qic.qkv, g_qic.a, &f) : vkc_attn_full(g_qic.qkv, g_qic.a, &f);} else {VkcEncAttn x = *at;x.S = n; x.q_off = at->q_off + r0 * at->kv_row; x.o_off = at->o_off + r0 * at->o_row;diff --git a/c/shaders/chain_attn_coop.comp b/c/shaders/chain_attn_coop.compnew file mode 100644index 00000000..a6751bab--- /dev/null+++ b/c/shaders/chain_attn_coop.comp@@ -0,0 +1,145 @@+#version 450+// chain_attn_full.comp's attention (vk_chain.c, vkc_attn_full with the matrix units; a+// diffusion step's image rows over every key) on cooperative matrices, f16 x f16 -> f32+// in 16 x 16 tiles: the same bindings, push constants and layout.+// o[o_off + i*o_row + h*HD + d] = sum_t softmax_t(scale * q_i . k_t) v_t[d]+// A workgroup of 256 threads (256 / SG subgroups) takes 64 query rows of one head and+// walks the keys twice, 32 at a time:+// pass 1, every row's max m and sum l of exp(score - m), the scores from the MMAs+// going through shared memory where the threads read them (4 threads a row);+// pass 2, the scores again, P = exp(score - m) / l as f16, then O += P V into the+// accumulators: m and l are final, so nothing is rescaled.+// Q (times scale), K, V and P are rounded to f16, so the result is the f32 attention's+// to f16 rounding (about 1e-3 relative), unlike chain_attn_full.comp's: the chain picks+// it for int8 weights only. Tiles are staged 16 x 16 contiguous (tile (a, b) at+// (a*NB + b)*256, element (row, col) at row*16 + col), Q once, K and V each key block;+// K as the B operand is a key x dim tile loaded column-major. Rows past S and keys past+// T are zeros, the keys masked out of the softmax. Specialization constants: HD (64 or+// 128) and SG (the subgroup size the pipeline requires, 32 or 64). Shared memory: 37 KB.+#extension GL_KHR_cooperative_matrix : require+#extension GL_KHR_memory_scope_semantics : require+#extension GL_KHR_shader_subgroup_basic : require+#extension GL_EXT_shader_explicit_arithmetic_types_float16 : require+#extension GL_EXT_control_flow_attributes : enable+layout(local_size_x = 256) in;+layout(constant_id = 0) const int HD = 128;+layout(constant_id = 1) const int SG = 64;+#define BR 64 // query rows a workgroup: 4 row tiles+#define BC 32 // keys a block: 2 key tiles+#define NDT (HD / 16) // dim tiles+#define NSUB (256 / SG) // subgroups+#define OT (4 * NDT / NSUB) // output tiles a subgroup++layout(std430, binding = 0) readonly buffer QKV { float x[]; };+layout(std430, binding = 1) writeonly buffer OB { float o[]; };+layout(push_constant) uniform P {+ int S, T, H, hd, q_off, q_row, k_off, v_off, kv_row, o_off, o_row;+ float scale;+} p;++shared float16_t sq[4 * NDT * 256]; // Q * scale, tiles (row tile, dim tile)+shared float16_t skv[2 * NDT * 256]; // K, then V, tiles (key tile, dim tile)+shared float16_t sp[4 * 2 * 256]; // P, tiles (row tile, key tile)+shared float ss[4 * 2 * 256]; // the block's scores, tiles (row tile, key tile); the output's staging+shared float red[BR * 4];++void stage_kv(int off, int kb, int hb, int t) {+ for (int e = t; e < BC * HD; e += 256) {+ int key = e / HD, d = e - key * HD, k = kb + key;+ float v = k < p.T ? x[off + k * p.kv_row + hb + d] : 0.0;+ skv[((key >> 4) * NDT + (d >> 4)) * 256 + (key & 15) * 16 + (d & 15)] = float16_t(v);+ }+}+void scores(int sgid) { // ss = Q K^T for the block+ for (int tile = sgid; tile < 8; tile += NSUB) {+ int ri = tile >> 1, kj = tile & 1;+ coopmat<float, gl_ScopeSubgroup, 16, 16, gl_MatrixUseAccumulator> c =+ coopmat<float, gl_ScopeSubgroup, 16, 16, gl_MatrixUseAccumulator>(0.0);+ [[unroll]] for (int dt = 0; dt < NDT; dt++) {+ coopmat<float16_t, gl_ScopeSubgroup, 16, 16, gl_MatrixUseA> a;+ coopmat<float16_t, gl_ScopeSubgroup, 16, 16, gl_MatrixUseB> b;+ coopMatLoad(a, sq, (ri * NDT + dt) * 256, 16, gl_CooperativeMatrixLayoutRowMajor);+ coopMatLoad(b, skv, (kj * NDT + dt) * 256, 16, gl_CooperativeMatrixLayoutColumnMajor);+ c = coopMatMulAdd(a, b, c);+ }+ coopMatStore(c, ss, (ri * 2 + kj) * 256, 16, gl_CooperativeMatrixLayoutRowMajor);+ }+}+float score_at(int row, int key, int kb) { // the block's score, masked past T+ return kb + key < p.T ? ss[((row >> 4) * 2 + (key >> 4)) * 256 + (row & 15) * 16 + (key & 15)] : -1e30;+}++void main() {+ int t = int(gl_LocalInvocationID.x), sgid = t / SG;+ int h = int(gl_WorkGroupID.y), r0 = int(gl_WorkGroupID.x) * BR, hb = h * HD;+ int row = t >> 2, part = t & 3; // the softmax: 4 threads a row, 8 keys each+ for (int e = t; e < BR * HD; e += 256) {+ int r = e / HD, d = e - r * HD, i = r0 + r;+ float v = i < p.S ? x[p.q_off + i * p.q_row + hb + d] * p.scale : 0.0;+ sq[((r >> 4) * NDT + (d >> 4)) * 256 + (r & 15) * 16 + (d & 15)] = float16_t(v);+ }+ // pass 1: each row's max and sum+ float m = -1e30, l = 0.0;+ for (int kb = 0; kb < p.T; kb += BC) {+ barrier();+ stage_kv(p.k_off, kb, hb, t);+ barrier();+ scores(sgid);+ barrier();+ float mt = -1e30;+ for (int j = 0; j < 8; j++) mt = max(mt, score_at(row, part * 8 + j, kb));+ red[row * 4 + part] = mt;+ barrier();+ float mn = max(max(red[row * 4], red[row * 4 + 1]), max(red[row * 4 + 2], red[row * 4 + 3]));+ mn = max(m, mn);+ float s = 0.0;+ for (int j = 0; j < 8; j++) { int k = part * 8 + j; s += kb + k < p.T ? exp(score_at(row, k, kb) - mn) : 0.0; }+ barrier();+ red[row * 4 + part] = s;+ barrier();+ l = l * exp(m - mn) + red[row * 4] + red[row * 4 + 1] + red[row * 4 + 2] + red[row * 4 + 3];+ m = mn;+ }+ float il = 1.0 / l;+ // pass 2: P V into the accumulators+ coopmat<float, gl_ScopeSubgroup, 16, 16, gl_MatrixUseAccumulator> acc[OT];+ [[unroll]] for (int u = 0; u < OT; u++) acc[u] = coopmat<float, gl_ScopeSubgroup, 16, 16, gl_MatrixUseAccumulator>(0.0);+ for (int kb = 0; kb < p.T; kb += BC) {+ barrier();+ stage_kv(p.k_off, kb, hb, t);+ barrier();+ scores(sgid);+ barrier();+ for (int j = 0; j < 8; j++) {+ int k = part * 8 + j;+ float pv = kb + k < p.T ? exp(score_at(row, k, kb) - m) * il : 0.0;+ sp[((row >> 4) * 2 + (k >> 4)) * 256 + (row & 15) * 16 + (k & 15)] = float16_t(pv);+ }+ barrier(); // K is read: V takes its place+ stage_kv(p.v_off, kb, hb, t);+ barrier();+ [[unroll]] for (int u = 0; u < OT; u++) {+ int ot = sgid + NSUB * u, ri = ot / NDT, dt = ot - ri * NDT;+ [[unroll]] for (int kj = 0; kj < 2; kj++) {+ coopmat<float16_t, gl_ScopeSubgroup, 16, 16, gl_MatrixUseA> a;+ coopmat<float16_t, gl_ScopeSubgroup, 16, 16, gl_MatrixUseB> b;+ coopMatLoad(a, sp, (ri * 2 + kj) * 256, 16, gl_CooperativeMatrixLayoutRowMajor);+ coopMatLoad(b, skv, (kj * NDT + dt) * 256, 16, gl_CooperativeMatrixLayoutRowMajor);+ acc[u] = coopMatMulAdd(a, b, acc[u]);+ }+ }+ }+ // the output, a row tile at a time through ss (16 rows x HD floats: HD / 16 tiles)+ for (int ri = 0; ri < 4; ri++) {+ barrier();+ [[unroll]] for (int u = 0; u < OT; u++) {+ int ot = sgid + NSUB * u;+ if (ot / NDT == ri) coopMatStore(acc[u], ss, (ot - ri * NDT) * 256, 16, gl_CooperativeMatrixLayoutRowMajor);+ }+ barrier();+ for (int e = t; e < 16 * HD; e += 256) {+ int r = e / HD, d = e - r * HD, i = r0 + ri * 16 + r;+ if (i < p.S) o[p.o_off + i * p.o_row + hb + d] = ss[(d >> 4) * 256 + r * 16 + (d & 15)];+ }+ }+}diff --git a/c/shaders/chain_attn_coop.spv b/c/shaders/chain_attn_coop.spvnew file mode 100644index 00000000..f47465cfBinary files /dev/null and b/c/shaders/chain_attn_coop.spv differdiff --git a/c/tests/test_vk_chain.c b/c/tests/test_vk_chain.cindex f4f83912..028f77fc 100644--- a/c/tests/test_vk_chain.c+++ b/c/tests/test_vk_chain.c@@ -321,7 +321,9 @@ static void test_ew_gate(void) {/* A diffusion step's attention (vkc_attn_full, Qwen-Image's DiT): S query rows, each over* all T key rows, token-major in one buffer (q, then k and v at their offsets), heads of* hd; S and T away from the shader's tiles of 64. */-static void test_attn_full(int S, int T, int H, int hd) {+static void test_attn_full_k(int S, int T, int H, int hd, int coop);+static void test_attn_full(int S, int T, int H, int hd) { test_attn_full_k(S, T, H, hd, 0); }+static void test_attn_full_k(int S, int T, int H, int hd, int coop) {int D = H * hd, qrow = D + 8, kvrow = D + 24, qoff = 4, koff = qoff + S * qrow + 16, voff = koff + T * kvrow + 8;size_t n = (size_t)voff + (size_t)T * kvrow + 8;float *x = fvec(n, 1.f), scale = 1.f / sqrtf((float)hd);@@ -343,9 +345,10 @@ static void test_attn_full(int S, int T, int H, int hd) {}VkcBuf *xb = up(x, n), *ob = vkc_buf(((size_t)S * D + 32) * 4, VKC_DOWN);VkcAttnFull p = {S, T, H, hd, qoff, qrow, koff, voff, kvrow, 32, D, scale};- vkc_begin(); int ok = vkc_attn_full(xb, ob, &p); ok = vkc_submit(1) && ok;+ vkc_begin(); int ok = coop ? vkc_attn_full_coop(xb, ob, &p) : vkc_attn_full(xb, ob, &p); ok = vkc_submit(1) && ok;double e = relerr((float *)vkc_ptr(ob) + 32, ref, (size_t)S * D, 1e-3);- CHECK(ok && e < 2e-5, "attn full S %d T %d H %d hd %d: err %.2e", S, T, H, hd, e);+ /* the matrix units' operands are f16: a bound of f16's rounding */+ CHECK(ok && e < (coop ? 4e-3 : 2e-5), "attn full%s S %d T %d H %d hd %d: err %.2e", coop ? " coop" : "", S, T, H, hd, e);vkc_free(xb); vkc_free(ob); free(x); free(ref); free(sc);}@@ -2766,6 +2769,11 @@ int main(int argc, char **argv) {test_attn_full(5, 9, 2, 32); test_attn_full(64, 64, 1, 128); test_attn_full(130, 200, 3, 128);test_attn_full(70, 129, 2, 64); test_attn_full(1, 300, 4, 128);printf("attn full done\n");+ if (vkc_attn_full_coop_ready(128)) { /* on the matrix units, where the device has them */+ test_attn_full_k(64, 64, 1, 128, 1); test_attn_full_k(130, 200, 3, 128, 1);+ test_attn_full_k(70, 129, 2, 64, 1); test_attn_full_k(1, 300, 4, 128, 1);+ printf("attn full coop done\n");+ } else printf("attn full coop: no matrix units here\n");test_ew_gate();test_vae_ops(5, 7, 3, 0, 0, 5); test_vae_ops(5, 7, 3, 1, 3, 4); test_vae_ops(4, 4, 16, 1, 0, 8);printf("vae ops done\n");diff --git a/c/vk_chain.c b/c/vk_chain.cindex 3f8db4f9..63b2d90c 100644--- a/c/vk_chain.c+++ b/c/vk_chain.c@@ -1395,11 +1395,13 @@ static struct { VkShaderModule mod; VkPipeline pipe; int tried; } g_ke[2];#define KE (g_ke[g_kd])/* chain_attn_full.comp (a diffusion step's attention): made on first use, a pipeline per* head dim (its specialization constant) */-static struct { VkShaderModule mod; VkPipeline pipe[3]; int tried; } g_kf[2];+static struct { VkShaderModule mod; VkPipeline pipe[3]; int tried; VkShaderModule cmod; VkPipeline cpipe[2]; int ctried; } g_kf[2];#define KF (g_kf[g_kd])static void kf_shutdown(void) {for (int i = 0; i < 3; i++) if (KF.pipe[i]) vkDestroyPipeline(KC.dev, KF.pipe[i], NULL);+ for (int i = 0; i < 2; i++) if (KF.cpipe[i]) vkDestroyPipeline(KC.dev, KF.cpipe[i], NULL);if (KF.mod) vkDestroyShaderModule(KC.dev, KF.mod, NULL);+ if (KF.cmod) vkDestroyShaderModule(KC.dev, KF.cmod, NULL);memset(&KF, 0, sizeof KF);}int vkc_attn_full(VkcBuf *qkv, VkcBuf *o, const VkcAttnFull *p) {@@ -1418,6 +1420,41 @@ int vkc_attn_full(VkcBuf *qkv, VkcBuf *o, const VkcAttnFull *p) {VkcBind bd[2] = {B(qkv, 0), B(o, 1)};return record(KF.pipe[k], bd, 2, p, sizeof *p, (uint32_t)((p->S + 63) / 64), (uint32_t)p->H, 1);}+/* The same on the matrix units (chain_attn_coop.comp): a device with cooperative matrices+ * at subgroups of 32 or 64, head dims 64 and 128; its pipeline requires that subgroup+ * size. 0: not here (nothing recorded). */+static VkPipeline kf_coop_pipe(int hd) {+ int k = hd == 128, sg = KC.core.coop_sg;+ if (KF.cpipe[k]) return KF.cpipe[k];+ if ((sg != 32 && sg != 64) || (hd != 64 && hd != 128)) return VK_NULL_HANDLE;+ if (!KF.cmod) {+ if (KF.ctried) return VK_NULL_HANDLE;+ KF.ctried = 1;+ if (!(KF.cmod = load_module(KC.core.spv_path, "chain_attn_coop.spv"))) return VK_NULL_HANDLE;+ }+ int32_t sv[2] = {hd, sg};+ VkSpecializationMapEntry me[2] = {{0, 0, 4}, {1, 4, 4}};+ VkSpecializationInfo si = {2, me, sizeof sv, sv};+ VkPipelineShaderStageRequiredSubgroupSizeCreateInfoEXT rss = {+ .sType = VK_STRUCTURE_TYPE_PIPELINE_SHADER_STAGE_REQUIRED_SUBGROUP_SIZE_CREATE_INFO_EXT,+ .requiredSubgroupSize = (uint32_t)sg};+ VkComputePipelineCreateInfo ci = {.sType = VK_STRUCTURE_TYPE_COMPUTE_PIPELINE_CREATE_INFO,+ .stage = {.sType = VK_STRUCTURE_TYPE_PIPELINE_SHADER_STAGE_CREATE_INFO, .pNext = &rss,+ .flags = VK_PIPELINE_SHADER_STAGE_CREATE_REQUIRE_FULL_SUBGROUPS_BIT_EXT,+ .stage = VK_SHADER_STAGE_COMPUTE_BIT, .module = KF.cmod, .pName = "main",+ .pSpecializationInfo = &si}, .layout = KC.pl};+ if (vkCreateComputePipelines(KC.dev, VK_NULL_HANDLE, 1, &ci, NULL, &KF.cpipe[k]) != VK_SUCCESS) KF.cpipe[k] = VK_NULL_HANDLE;+ return KF.cpipe[k];+}+int vkc_attn_full_coop(VkcBuf *qkv, VkcBuf *o, const VkcAttnFull *p) {+ if (!vkc_ready() || p->S < 1 || p->T < 1 || p->H < 1 || (int64_t)p->S > 65535LL * 64 || p->H > 65535) return 0;+ VkPipeline pipe = kf_coop_pipe(p->hd);+ if (!pipe) return 0;+ KC.kind = PK_ATTN;+ VkcBind bd[2] = {B(qkv, 0), B(o, 1)};+ return record(pipe, bd, 2, p, sizeof *p, (uint32_t)((p->S + 63) / 64), (uint32_t)p->H, 1);+}+int vkc_attn_full_coop_ready(int hd) { return vkc_ready() && kf_coop_pipe(hd) != VK_NULL_HANDLE; }int vkc_attn_full_ready(void) {if (!vkc_ready()) return 0;if (!KF.mod && !KF.tried) { KF.tried = 1; KF.mod = load_module(KC.core.spv_path, "chain_attn_full.spv"); }diff --git a/c/vk_chain.h b/c/vk_chain.hindex 08815198..2ce085ee 100644--- a/c/vk_chain.h+++ b/c/vk_chain.h@@ -686,6 +686,11 @@ int vkc_enc_rel(VkcBuf *x, VkcBuf *pt, VkcBuf *y, const VkcEncRel *p);typedef struct { int S, T, H, hd, q_off, q_row, k_off, v_off, kv_row, o_off, o_row; float scale; } VkcAttnFull;int vkc_attn_full(VkcBuf *qkv, VkcBuf *o, const VkcAttnFull *p);int vkc_attn_full_ready(void);+/* The same on the matrix units (chain_attn_coop.comp, cooperative matrices at subgroups of+ * 32 or 64, hd 64 or 128): Q, K, V and P rounded to f16, the result the f32 one's to about+ * 1e-3. 0: not on this device (nothing recorded). */+int vkc_attn_full_coop(VkcBuf *qkv, VkcBuf *o, const VkcAttnFull *p);+int vkc_attn_full_coop_ready(int hd);/* chain_vae.comp: Qwen-Image's VAE decoder (qwenimage_vae_vk.h), the two steps of its* convolutions that are not a GEMM; maps channel-last [H][W][C]. mode 0: the 3x3 taps of