Skip to content

Run frontier MoE models on hardware you already own — pure C, zero deps, experts streamed from disk. Tiny engine, immense model. 🐦

C

vulkan chain: the diffusion attention on the matrix units, opt-in

JustVugg10c41230c583
diff --git a/c/Makefile b/c/Makefile
index 83a32933..8b755323 100644
--- a/c/Makefile
+++ b/c/Makefile
@@ -745,7 +745,7 @@ VK_SPV += shaders/chain_enc.spv
VK_SPV += shaders/chain_attn_flash.spv
# a diffusion step's attention, every query row over every key row (made on first use;
# qwenimage_chain.h)
-VK_SPV += shaders/chain_attn_full.spv
+VK_SPV += shaders/chain_attn_full.spv shaders/chain_attn_coop.spv
# Qwen-Image's VAE decoder: a band's 3x3 taps and the DupUp3D shortcut (made on first use)
VK_SPV += shaders/chain_vae.spv
endif
diff --git a/c/qwenimage_chain.h b/c/qwenimage_chain.h
index 7863174b..8f1e401e 100644
--- a/c/qwenimage_chain.h
+++ b/c/qwenimage_chain.h
@@ -55,6 +55,7 @@ typedef struct {
unsigned long long steps;
double wait_ms;
int resident, streamed; /* blocks the device keeps (0..resident-1), blocks uploaded each step */
+ int coop_attn; /* the attention on the matrix units (int8 weights) */
int slot[2]; /* the streamed block in each of the two places (-1: none) */
unsigned long long ser[2]; /* the frame that last read it */
unsigned long long stream_bytes;
@@ -108,6 +109,12 @@ static void qic_plan(Dit *d) {
if (R > L) R = L;
}
g_qic.resident = R; g_qic.streamed = L - R; g_qic.slot[0] = g_qic.slot[1] = -1;
+ const char *ae = getenv("COLI_VK_QI_ATTN");
+ /* opt-in: on a Radeon 780M the f32 tiles beat the matrix units (a 1024x1024 step's
+ * attention 17.9 s against 32.5: RDNA3's f16 MMA is twice its f32 rate, and the two
+ * passes cost more than that); a card whose matrix units outrun its f32 by more may
+ * gain, not measured yet */
+ g_qic.coop_attn = ae && !strcmp(ae, "coop") && d->B[0].q.m.fmt == QI_I8;
if (R == L) return;
for (int l = R; l < L; l++) { /* the per-matrix path leaves them on the CPU */
Lin *m[7]; qic_block_mats(&d->B[l], m);
@@ -218,15 +225,17 @@ static int qic_attn(const VkcEncAttn *at, int N, int L) {
long long b = vkc_attn_slice_budget(), per = (long long)(L + N) * at->H * at->hd;
int rr = b > 0 ? (int)(b / per / 16 * 16) : N;
if (rr < 16) rr = 16;
- /* chain_attn_full where it loads (head dims 32, 64 and 128), the encoders'
- * attention over row ranges otherwise */
- int full = (at->hd == 32 || at->hd == 64 || at->hd == 128) && vkc_attn_full_ready(), ok = 1;
+ /* on the matrix units with COLI_VK_QI_ATTN=coop and int8 weights (their f16
+ * operands cost less than the weights' own rounding), else chain_attn_full where it
+ * loads (head dims 32, 64 and 128), else the encoders' attention over row ranges */
+ int coop = g_qic.coop_attn && vkc_attn_full_coop_ready(at->hd);
+ int full = coop || ((at->hd == 32 || at->hd == 64 || at->hd == 128) && vkc_attn_full_ready()), ok = 1;
for (int r0 = 0; ok && r0 < N; r0 += rr) {
int n = N - r0 < rr ? N - r0 : rr;
if (full) {
VkcAttnFull f = {n, L + N, at->H, at->hd, at->q_off + r0 * at->kv_row, at->kv_row, at->k_off, at->v_off,
at->kv_row, at->o_off + r0 * at->o_row, at->o_row, at->scale};
- ok = vkc_attn_full(g_qic.qkv, g_qic.a, &f);
+ ok = coop ? vkc_attn_full_coop(g_qic.qkv, g_qic.a, &f) : vkc_attn_full(g_qic.qkv, g_qic.a, &f);
} else {
VkcEncAttn x = *at;
x.S = n; x.q_off = at->q_off + r0 * at->kv_row; x.o_off = at->o_off + r0 * at->o_row;
diff --git a/c/shaders/chain_attn_coop.comp b/c/shaders/chain_attn_coop.comp
new file mode 100644
index 00000000..a6751bab
--- /dev/null
+++ b/c/shaders/chain_attn_coop.comp
@@ -0,0 +1,145 @@
+#version 450
+// chain_attn_full.comp's attention (vk_chain.c, vkc_attn_full with the matrix units; a
+// diffusion step's image rows over every key) on cooperative matrices, f16 x f16 -> f32
+// in 16 x 16 tiles: the same bindings, push constants and layout.
+// o[o_off + i*o_row + h*HD + d] = sum_t softmax_t(scale * q_i . k_t) v_t[d]
+// A workgroup of 256 threads (256 / SG subgroups) takes 64 query rows of one head and
+// walks the keys twice, 32 at a time:
+// pass 1, every row's max m and sum l of exp(score - m), the scores from the MMAs
+// going through shared memory where the threads read them (4 threads a row);
+// pass 2, the scores again, P = exp(score - m) / l as f16, then O += P V into the
+// accumulators: m and l are final, so nothing is rescaled.
+// Q (times scale), K, V and P are rounded to f16, so the result is the f32 attention's
+// to f16 rounding (about 1e-3 relative), unlike chain_attn_full.comp's: the chain picks
+// it for int8 weights only. Tiles are staged 16 x 16 contiguous (tile (a, b) at
+// (a*NB + b)*256, element (row, col) at row*16 + col), Q once, K and V each key block;
+// K as the B operand is a key x dim tile loaded column-major. Rows past S and keys past
+// T are zeros, the keys masked out of the softmax. Specialization constants: HD (64 or
+// 128) and SG (the subgroup size the pipeline requires, 32 or 64). Shared memory: 37 KB.
+#extension GL_KHR_cooperative_matrix : require
+#extension GL_KHR_memory_scope_semantics : require
+#extension GL_KHR_shader_subgroup_basic : require
+#extension GL_EXT_shader_explicit_arithmetic_types_float16 : require
+#extension GL_EXT_control_flow_attributes : enable
+layout(local_size_x = 256) in;
+layout(constant_id = 0) const int HD = 128;
+layout(constant_id = 1) const int SG = 64;
+#define BR 64 // query rows a workgroup: 4 row tiles
+#define BC 32 // keys a block: 2 key tiles
+#define NDT (HD / 16) // dim tiles
+#define NSUB (256 / SG) // subgroups
+#define OT (4 * NDT / NSUB) // output tiles a subgroup
+
+layout(std430, binding = 0) readonly buffer QKV { float x[]; };
+layout(std430, binding = 1) writeonly buffer OB { float o[]; };
+layout(push_constant) uniform P {
+ int S, T, H, hd, q_off, q_row, k_off, v_off, kv_row, o_off, o_row;
+ float scale;
+} p;
+
+shared float16_t sq[4 * NDT * 256]; // Q * scale, tiles (row tile, dim tile)
+shared float16_t skv[2 * NDT * 256]; // K, then V, tiles (key tile, dim tile)
+shared float16_t sp[4 * 2 * 256]; // P, tiles (row tile, key tile)
+shared float ss[4 * 2 * 256]; // the block's scores, tiles (row tile, key tile); the output's staging
+shared float red[BR * 4];
+
+void stage_kv(int off, int kb, int hb, int t) {
+ for (int e = t; e < BC * HD; e += 256) {
+ int key = e / HD, d = e - key * HD, k = kb + key;
+ float v = k < p.T ? x[off + k * p.kv_row + hb + d] : 0.0;
+ skv[((key >> 4) * NDT + (d >> 4)) * 256 + (key & 15) * 16 + (d & 15)] = float16_t(v);
+ }
+}
+void scores(int sgid) { // ss = Q K^T for the block
+ for (int tile = sgid; tile < 8; tile += NSUB) {
+ int ri = tile >> 1, kj = tile & 1;
+ coopmat<float, gl_ScopeSubgroup, 16, 16, gl_MatrixUseAccumulator> c =
+ coopmat<float, gl_ScopeSubgroup, 16, 16, gl_MatrixUseAccumulator>(0.0);
+ [[unroll]] for (int dt = 0; dt < NDT; dt++) {
+ coopmat<float16_t, gl_ScopeSubgroup, 16, 16, gl_MatrixUseA> a;
+ coopmat<float16_t, gl_ScopeSubgroup, 16, 16, gl_MatrixUseB> b;
+ coopMatLoad(a, sq, (ri * NDT + dt) * 256, 16, gl_CooperativeMatrixLayoutRowMajor);
+ coopMatLoad(b, skv, (kj * NDT + dt) * 256, 16, gl_CooperativeMatrixLayoutColumnMajor);
+ c = coopMatMulAdd(a, b, c);
+ }
+ coopMatStore(c, ss, (ri * 2 + kj) * 256, 16, gl_CooperativeMatrixLayoutRowMajor);
+ }
+}
+float score_at(int row, int key, int kb) { // the block's score, masked past T
+ return kb + key < p.T ? ss[((row >> 4) * 2 + (key >> 4)) * 256 + (row & 15) * 16 + (key & 15)] : -1e30;
+}
+
+void main() {
+ int t = int(gl_LocalInvocationID.x), sgid = t / SG;
+ int h = int(gl_WorkGroupID.y), r0 = int(gl_WorkGroupID.x) * BR, hb = h * HD;
+ int row = t >> 2, part = t & 3; // the softmax: 4 threads a row, 8 keys each
+ for (int e = t; e < BR * HD; e += 256) {
+ int r = e / HD, d = e - r * HD, i = r0 + r;
+ float v = i < p.S ? x[p.q_off + i * p.q_row + hb + d] * p.scale : 0.0;
+ sq[((r >> 4) * NDT + (d >> 4)) * 256 + (r & 15) * 16 + (d & 15)] = float16_t(v);
+ }
+ // pass 1: each row's max and sum
+ float m = -1e30, l = 0.0;
+ for (int kb = 0; kb < p.T; kb += BC) {
+ barrier();
+ stage_kv(p.k_off, kb, hb, t);
+ barrier();
+ scores(sgid);
+ barrier();
+ float mt = -1e30;
+ for (int j = 0; j < 8; j++) mt = max(mt, score_at(row, part * 8 + j, kb));
+ red[row * 4 + part] = mt;
+ barrier();
+ float mn = max(max(red[row * 4], red[row * 4 + 1]), max(red[row * 4 + 2], red[row * 4 + 3]));
+ mn = max(m, mn);
+ float s = 0.0;
+ for (int j = 0; j < 8; j++) { int k = part * 8 + j; s += kb + k < p.T ? exp(score_at(row, k, kb) - mn) : 0.0; }
+ barrier();
+ red[row * 4 + part] = s;
+ barrier();
+ l = l * exp(m - mn) + red[row * 4] + red[row * 4 + 1] + red[row * 4 + 2] + red[row * 4 + 3];
+ m = mn;
+ }
+ float il = 1.0 / l;
+ // pass 2: P V into the accumulators
+ coopmat<float, gl_ScopeSubgroup, 16, 16, gl_MatrixUseAccumulator> acc[OT];
+ [[unroll]] for (int u = 0; u < OT; u++) acc[u] = coopmat<float, gl_ScopeSubgroup, 16, 16, gl_MatrixUseAccumulator>(0.0);
+ for (int kb = 0; kb < p.T; kb += BC) {
+ barrier();
+ stage_kv(p.k_off, kb, hb, t);
+ barrier();
+ scores(sgid);
+ barrier();
+ for (int j = 0; j < 8; j++) {
+ int k = part * 8 + j;
+ float pv = kb + k < p.T ? exp(score_at(row, k, kb) - m) * il : 0.0;
+ sp[((row >> 4) * 2 + (k >> 4)) * 256 + (row & 15) * 16 + (k & 15)] = float16_t(pv);
+ }
+ barrier(); // K is read: V takes its place
+ stage_kv(p.v_off, kb, hb, t);
+ barrier();
+ [[unroll]] for (int u = 0; u < OT; u++) {
+ int ot = sgid + NSUB * u, ri = ot / NDT, dt = ot - ri * NDT;
+ [[unroll]] for (int kj = 0; kj < 2; kj++) {
+ coopmat<float16_t, gl_ScopeSubgroup, 16, 16, gl_MatrixUseA> a;
+ coopmat<float16_t, gl_ScopeSubgroup, 16, 16, gl_MatrixUseB> b;
+ coopMatLoad(a, sp, (ri * 2 + kj) * 256, 16, gl_CooperativeMatrixLayoutRowMajor);
+ coopMatLoad(b, skv, (kj * NDT + dt) * 256, 16, gl_CooperativeMatrixLayoutRowMajor);
+ acc[u] = coopMatMulAdd(a, b, acc[u]);
+ }
+ }
+ }
+ // the output, a row tile at a time through ss (16 rows x HD floats: HD / 16 tiles)
+ for (int ri = 0; ri < 4; ri++) {
+ barrier();
+ [[unroll]] for (int u = 0; u < OT; u++) {
+ int ot = sgid + NSUB * u;
+ if (ot / NDT == ri) coopMatStore(acc[u], ss, (ot - ri * NDT) * 256, 16, gl_CooperativeMatrixLayoutRowMajor);
+ }
+ barrier();
+ for (int e = t; e < 16 * HD; e += 256) {
+ int r = e / HD, d = e - r * HD, i = r0 + ri * 16 + r;
+ if (i < p.S) o[p.o_off + i * p.o_row + hb + d] = ss[(d >> 4) * 256 + r * 16 + (d & 15)];
+ }
+ }
+}
diff --git a/c/shaders/chain_attn_coop.spv b/c/shaders/chain_attn_coop.spv
new file mode 100644
index 00000000..f47465cf
Binary files /dev/null and b/c/shaders/chain_attn_coop.spv differ
diff --git a/c/tests/test_vk_chain.c b/c/tests/test_vk_chain.c
index f4f83912..028f77fc 100644
--- a/c/tests/test_vk_chain.c
+++ b/c/tests/test_vk_chain.c
@@ -321,7 +321,9 @@ static void test_ew_gate(void) {
/* A diffusion step's attention (vkc_attn_full, Qwen-Image's DiT): S query rows, each over
* all T key rows, token-major in one buffer (q, then k and v at their offsets), heads of
* hd; S and T away from the shader's tiles of 64. */
-static void test_attn_full(int S, int T, int H, int hd) {
+static void test_attn_full_k(int S, int T, int H, int hd, int coop);
+static void test_attn_full(int S, int T, int H, int hd) { test_attn_full_k(S, T, H, hd, 0); }
+static void test_attn_full_k(int S, int T, int H, int hd, int coop) {
int D = H * hd, qrow = D + 8, kvrow = D + 24, qoff = 4, koff = qoff + S * qrow + 16, voff = koff + T * kvrow + 8;
size_t n = (size_t)voff + (size_t)T * kvrow + 8;
float *x = fvec(n, 1.f), scale = 1.f / sqrtf((float)hd);
@@ -343,9 +345,10 @@ static void test_attn_full(int S, int T, int H, int hd) {
}
VkcBuf *xb = up(x, n), *ob = vkc_buf(((size_t)S * D + 32) * 4, VKC_DOWN);
VkcAttnFull p = {S, T, H, hd, qoff, qrow, koff, voff, kvrow, 32, D, scale};
- vkc_begin(); int ok = vkc_attn_full(xb, ob, &p); ok = vkc_submit(1) && ok;
+ vkc_begin(); int ok = coop ? vkc_attn_full_coop(xb, ob, &p) : vkc_attn_full(xb, ob, &p); ok = vkc_submit(1) && ok;
double e = relerr((float *)vkc_ptr(ob) + 32, ref, (size_t)S * D, 1e-3);
- CHECK(ok && e < 2e-5, "attn full S %d T %d H %d hd %d: err %.2e", S, T, H, hd, e);
+ /* the matrix units' operands are f16: a bound of f16's rounding */
+ CHECK(ok && e < (coop ? 4e-3 : 2e-5), "attn full%s S %d T %d H %d hd %d: err %.2e", coop ? " coop" : "", S, T, H, hd, e);
vkc_free(xb); vkc_free(ob); free(x); free(ref); free(sc);
}
@@ -2766,6 +2769,11 @@ int main(int argc, char **argv) {
test_attn_full(5, 9, 2, 32); test_attn_full(64, 64, 1, 128); test_attn_full(130, 200, 3, 128);
test_attn_full(70, 129, 2, 64); test_attn_full(1, 300, 4, 128);
printf("attn full done\n");
+ if (vkc_attn_full_coop_ready(128)) { /* on the matrix units, where the device has them */
+ test_attn_full_k(64, 64, 1, 128, 1); test_attn_full_k(130, 200, 3, 128, 1);
+ test_attn_full_k(70, 129, 2, 64, 1); test_attn_full_k(1, 300, 4, 128, 1);
+ printf("attn full coop done\n");
+ } else printf("attn full coop: no matrix units here\n");
test_ew_gate();
test_vae_ops(5, 7, 3, 0, 0, 5); test_vae_ops(5, 7, 3, 1, 3, 4); test_vae_ops(4, 4, 16, 1, 0, 8);
printf("vae ops done\n");
diff --git a/c/vk_chain.c b/c/vk_chain.c
index 3f8db4f9..63b2d90c 100644
--- a/c/vk_chain.c
+++ b/c/vk_chain.c
@@ -1395,11 +1395,13 @@ static struct { VkShaderModule mod; VkPipeline pipe; int tried; } g_ke[2];
#define KE (g_ke[g_kd])
/* chain_attn_full.comp (a diffusion step's attention): made on first use, a pipeline per
* head dim (its specialization constant) */
-static struct { VkShaderModule mod; VkPipeline pipe[3]; int tried; } g_kf[2];
+static struct { VkShaderModule mod; VkPipeline pipe[3]; int tried; VkShaderModule cmod; VkPipeline cpipe[2]; int ctried; } g_kf[2];
#define KF (g_kf[g_kd])
static void kf_shutdown(void) {
for (int i = 0; i < 3; i++) if (KF.pipe[i]) vkDestroyPipeline(KC.dev, KF.pipe[i], NULL);
+ for (int i = 0; i < 2; i++) if (KF.cpipe[i]) vkDestroyPipeline(KC.dev, KF.cpipe[i], NULL);
if (KF.mod) vkDestroyShaderModule(KC.dev, KF.mod, NULL);
+ if (KF.cmod) vkDestroyShaderModule(KC.dev, KF.cmod, NULL);
memset(&KF, 0, sizeof KF);
}
int vkc_attn_full(VkcBuf *qkv, VkcBuf *o, const VkcAttnFull *p) {
@@ -1418,6 +1420,41 @@ int vkc_attn_full(VkcBuf *qkv, VkcBuf *o, const VkcAttnFull *p) {
VkcBind bd[2] = {B(qkv, 0), B(o, 1)};
return record(KF.pipe[k], bd, 2, p, sizeof *p, (uint32_t)((p->S + 63) / 64), (uint32_t)p->H, 1);
}
+/* The same on the matrix units (chain_attn_coop.comp): a device with cooperative matrices
+ * at subgroups of 32 or 64, head dims 64 and 128; its pipeline requires that subgroup
+ * size. 0: not here (nothing recorded). */
+static VkPipeline kf_coop_pipe(int hd) {
+ int k = hd == 128, sg = KC.core.coop_sg;
+ if (KF.cpipe[k]) return KF.cpipe[k];
+ if ((sg != 32 && sg != 64) || (hd != 64 && hd != 128)) return VK_NULL_HANDLE;
+ if (!KF.cmod) {
+ if (KF.ctried) return VK_NULL_HANDLE;
+ KF.ctried = 1;
+ if (!(KF.cmod = load_module(KC.core.spv_path, "chain_attn_coop.spv"))) return VK_NULL_HANDLE;
+ }
+ int32_t sv[2] = {hd, sg};
+ VkSpecializationMapEntry me[2] = {{0, 0, 4}, {1, 4, 4}};
+ VkSpecializationInfo si = {2, me, sizeof sv, sv};
+ VkPipelineShaderStageRequiredSubgroupSizeCreateInfoEXT rss = {
+ .sType = VK_STRUCTURE_TYPE_PIPELINE_SHADER_STAGE_REQUIRED_SUBGROUP_SIZE_CREATE_INFO_EXT,
+ .requiredSubgroupSize = (uint32_t)sg};
+ VkComputePipelineCreateInfo ci = {.sType = VK_STRUCTURE_TYPE_COMPUTE_PIPELINE_CREATE_INFO,
+ .stage = {.sType = VK_STRUCTURE_TYPE_PIPELINE_SHADER_STAGE_CREATE_INFO, .pNext = &rss,
+ .flags = VK_PIPELINE_SHADER_STAGE_CREATE_REQUIRE_FULL_SUBGROUPS_BIT_EXT,
+ .stage = VK_SHADER_STAGE_COMPUTE_BIT, .module = KF.cmod, .pName = "main",
+ .pSpecializationInfo = &si}, .layout = KC.pl};
+ if (vkCreateComputePipelines(KC.dev, VK_NULL_HANDLE, 1, &ci, NULL, &KF.cpipe[k]) != VK_SUCCESS) KF.cpipe[k] = VK_NULL_HANDLE;
+ return KF.cpipe[k];
+}
+int vkc_attn_full_coop(VkcBuf *qkv, VkcBuf *o, const VkcAttnFull *p) {
+ if (!vkc_ready() || p->S < 1 || p->T < 1 || p->H < 1 || (int64_t)p->S > 65535LL * 64 || p->H > 65535) return 0;
+ VkPipeline pipe = kf_coop_pipe(p->hd);
+ if (!pipe) return 0;
+ KC.kind = PK_ATTN;
+ VkcBind bd[2] = {B(qkv, 0), B(o, 1)};
+ return record(pipe, bd, 2, p, sizeof *p, (uint32_t)((p->S + 63) / 64), (uint32_t)p->H, 1);
+}
+int vkc_attn_full_coop_ready(int hd) { return vkc_ready() && kf_coop_pipe(hd) != VK_NULL_HANDLE; }
int vkc_attn_full_ready(void) {
if (!vkc_ready()) return 0;
if (!KF.mod && !KF.tried) { KF.tried = 1; KF.mod = load_module(KC.core.spv_path, "chain_attn_full.spv"); }
diff --git a/c/vk_chain.h b/c/vk_chain.h
index 08815198..2ce085ee 100644
--- a/c/vk_chain.h
+++ b/c/vk_chain.h
@@ -686,6 +686,11 @@ int vkc_enc_rel(VkcBuf *x, VkcBuf *pt, VkcBuf *y, const VkcEncRel *p);
typedef struct { int S, T, H, hd, q_off, q_row, k_off, v_off, kv_row, o_off, o_row; float scale; } VkcAttnFull;
int vkc_attn_full(VkcBuf *qkv, VkcBuf *o, const VkcAttnFull *p);
int vkc_attn_full_ready(void);
+/* The same on the matrix units (chain_attn_coop.comp, cooperative matrices at subgroups of
+ * 32 or 64, hd 64 or 128): Q, K, V and P rounded to f16, the result the f32 one's to about
+ * 1e-3. 0: not on this device (nothing recorded). */
+int vkc_attn_full_coop(VkcBuf *qkv, VkcBuf *o, const VkcAttnFull *p);
+int vkc_attn_full_coop_ready(int hd);
/* chain_vae.comp: Qwen-Image's VAE decoder (qwenimage_vae_vk.h), the two steps of its
* convolutions that are not a GEMM; maps channel-last [H][W][C]. mode 0: the 3x3 taps of