Run frontier MoE models on hardware you already own — pure C, zero deps, experts streamed from disk. Tiny engine, immense model. 🐦
C
qwenimage chain: stream the blocks a small card does not hold
JustVugg22c2799a565a
diff --git a/c/qwenimage.c b/c/qwenimage.cindex f6d41cc9..13627fea 100644--- a/c/qwenimage.c+++ b/c/qwenimage.c@@ -541,6 +541,9 @@ static void dit_config(Dit *d, const char *model){}}+#ifdef COLI_VULKAN+static void qic_plan(Dit *d); /* qwenimage_chain.h: the blocks the device keeps */+#endifstatic void dit_load(Dit *d, const char *model){if (d->loaded) return;char dir[2048]; snprintf(dir, sizeof dir, "%s/transformer", model);@@ -588,7 +591,10 @@ static void dit_load(Dit *d, const char *model){now_s() - t0);#ifdef COLI_VULKAN/* after the DiT's weights, once: a missing device costs one line */- if (!g_vk_tried) { g_vk_tried = 1; g_vk_ready = coli_vk_init_env("qwenimage"); }+ if (!g_vk_tried) {+ g_vk_tried = 1; g_vk_ready = coli_vk_init_env("qwenimage");+ if (g_vk_ready) qic_plan(d); /* before the prompt's prefix uploads anything */+ }#endif}@@ -771,7 +777,7 @@ static void dit_step_free(DitStep *s){* device's decode fails) the CPU's. */static int qi_vae_decode(QiVae *v, const float *z, int h, int w, uint8_t *rgba, float *out_f){#ifdef COLI_VULKAN- if (g_qic_on > 0 && !g_qic.failed && qvv_decode(v, z, h, w, rgba, out_f) == 0) return 0;+ if (g_qic_on > 0 && !g_qic.failed && !g_qic.streamed && qvv_decode(v, z, h, w, rgba, out_f) == 0) return 0;#endifreturn qiv_decode(v, z, h, w, rgba, out_f);}@@ -1462,6 +1468,7 @@ static void usage(void){" COLI_IMG_ACT8=0 f32 activations in the DiT (default where VNNI exists: int8, about 2x per step)\n"" COLI_IMG_TE=resident|stage keep the text encoder loaded between prompts (serve default: resident)\n"" COLI_IMG_LOAD_THREADS=n layers loaded at once (default 4)\n"+ " COLI_VK_QI_RESIDENT=n transformer blocks kept on the GPU (default: what its free memory holds)\n"" COLI_VULKAN=1 (a VK=1 build): the transformer on the GPU; COLI_VK_CHAIN=0 keeps each step's blocks\n"" off the device chain (the matrices one by one)\n");}diff --git a/c/qwenimage_chain.h b/c/qwenimage_chain.hindex 9ca23052..7863174b 100644--- a/c/qwenimage_chain.h+++ b/c/qwenimage_chain.h@@ -24,11 +24,21 @@* tiles, the softmax online), so a prediction agrees with the CPU's to rounding: the* oracle (tests/vulkan_engines.sh, mimo-qwenimage) holds every stage to its tolerance.*+ * A device whose free memory does not hold every block (an 8 GB card: the int8 DiT is+ * 7.1 GB) keeps blocks 0..R-1 and uploads the others each step, two at a time: block l+ * goes up while block l-1 runs, into the place block l-2 leaves once its frame is+ * through (qic_plan decides R when the weights load, from the device's free memory less+ * the steps' buffers and COLI_VK_TIER_RESERVE_GB; COLI_VK_QI_RESIDENT=n forces it). The+ * streamed blocks' matrices then stay off the per-matrix path (the prompt's prefix, a+ * step the chain does not take: on the CPU), so nothing asks the device for more than it+ * has, and the VAE decodes on the CPU.+ ** Declines (the CPU's path runs, its matrices on the device one by one as before):* COLI_VK_CHAIN off, a matrix the device does not take, the device's memory refusing a* buffer. A device lost in a step: the step runs again on the CPU, and the CPU runs from* there (the latents are the host's). */#include "vk_chain.h"+#include "vk_kvsplit.h" /* vkc_dev_avail */typedef struct {int ok, failed;@@ -44,6 +54,10 @@ typedef struct {VkcBuf **pk, **pv; /* per block, the prompt's keys (permuted) and values, [L][D] */unsigned long long steps;double wait_ms;+ int resident, streamed; /* blocks the device keeps (0..resident-1), blocks uploaded each step */+ int slot[2]; /* the streamed block in each of the two places (-1: none) */+ unsigned long long ser[2]; /* the frame that last read it */+ unsigned long long stream_bytes;} QiChain;static QiChain g_qic;@@ -63,10 +77,81 @@ static void qic_off(const char *why) {g_qic.failed = 1;}-/* The weights and the norms, once: the same device copies the per-matrix path uses. */+static void qic_block_mats(DitBlock *B, Lin **m) {+ m[0] = &B->q; m[1] = &B->k; m[2] = &B->v; m[3] = &B->o; m[4] = &B->gate; m[5] = &B->proj; m[6] = &B->out;+}+static size_t qic_block_bytes(DitBlock *B) {+ Lin *m[7]; qic_block_mats(B, m);+ size_t b = 0;+ for (int k = 0; k < 7; k++) {+ int fmt = m[k]->m.fmt == QI_I8 ? 1 : m[k]->m.fmt == QI_BF16 ? 11 : 10;+ b += coli_vk_tensor_payload(fmt, m[k]->m.K, m[k]->m.N, 0) + coli_vk_buffer_alignment();+ }+ return b;+}+/* When the weights load, before anything goes up: whether the chain runs, and how many+ * blocks the device keeps (all, or what its free memory holds after a 1024x1024 step's+ * buffers, a prompt of 256 tokens' keys and values and COLI_VK_TIER_RESERVE_GB). */+static void qic_plan(Dit *d) {+ g_qic_on = vkc_init() && coli_vk_chain_decide("qwenimage", -1, COLI_VK_CHAIN_ON) == COLI_VK_CHAIN_ON;+ int L = d->layers, R = L;+ size_t block = qic_block_bytes(&d->B[0]), avail = 0;+ const char *e = getenv("COLI_VK_QI_RESIDENT");+ if (e && *e) R = atoi(e) < 0 ? 0 : atoi(e) > L ? L : atoi(e);+ else if (coli_vk_dense()) {+ size_t N = 4096, T = N + 256, D = d->dim, M = d->mlp;+ size_t work = 4 * (3 * N * D + 2 * N * M + (N + 2 * T) * D + 2 * (size_t)L * 256 * D) + 2 * block;+ avail = vkc_dev_avail();+ size_t room = avail / 10 * 8, need = vkc_fit_reserve() + work;+ room = room > need ? room - need : 0;+ R = block ? (int)(room / block) : L;+ if (R > L) R = L;+ }+ g_qic.resident = R; g_qic.streamed = L - R; g_qic.slot[0] = g_qic.slot[1] = -1;+ if (R == L) return;+ for (int l = R; l < L; l++) { /* the per-matrix path leaves them on the CPU */+ Lin *m[7]; qic_block_mats(&d->B[l], m);+ for (int k = 0; k < 7; k++) m[k]->gpu = 0;+ }+ char why[96];+ if (e && *e) snprintf(why, sizeof why, "COLI_VK_QI_RESIDENT=%s", e);+ else snprintf(why, sizeof why, "%.1f GiB free on the device", avail / 1073741824.0);+ if (g_qic_on)+ fprintf(stderr, "[VK] qwenimage: %d of %d blocks stay on the device (%s); the other %d go up each step, "+ "the VAE decodes on the CPU\n", R, L, why, L - R);+ else+ fprintf(stderr, "[VK] qwenimage: %d of %d blocks' matrices on the device (%s); the other %d run on the CPU\n",+ R, L, why, L - R);+}++/* A streamed block's matrices up, into the place block l-2 leaves once its frame is+ * through */+static void qic_unstream(Dit *d, int s) {+ if (g_qic.slot[s] < 0) return;+ Lin *m[7]; qic_block_mats(&d->B[g_qic.slot[s]], m);+ for (int k = 0; k < 7; k++) if (m[k]->vk) { coli_vk_tensor_free((ColiVkTensor *)m[k]->vk); m[k]->vk = NULL; }+ g_qic.slot[s] = -1;+}+static int qic_stream(Dit *d, int l) {+ int s = l & 1;+ if (g_qic.slot[s] >= 0) {+ if (!vkc_wait_serial(g_qic.ser[s])) return 0;+ qic_unstream(d, s);+ }+ Lin *m[7]; qic_block_mats(&d->B[l], m);+ g_qic.slot[s] = l;+ for (int k = 0; k < 7; k++) {+ if (!qic_tensor(m[k])) return 0;+ g_qic.stream_bytes += coli_vk_tensor_bytes((ColiVkTensor *)m[k]->vk);+ }+ return 1;+}++/* The weights and the norms, once: the same device copies the per-matrix path uses+ * (the blocks the device keeps; the streamed ones go up each step). */static int qic_setup(Dit *d) {int H = d->hd, Lr = d->layers;- for (int l = 0; l < Lr; l++) {+ for (int l = 0; l < g_qic.resident; l++) {DitBlock *B = &d->B[l];Lin *m[7] = {&B->q, &B->k, &B->v, &B->o, &B->gate, &B->proj, &B->out};for (int k = 0; k < 7; k++)@@ -86,7 +171,7 @@ static int qic_setup(Dit *d) {g_qic.mod = vkc_buf(5 * (size_t)d->dim * sizeof(float), VKC_UP);if (!ok || !g_qic.pk || !g_qic.pv || !g_qic.mod) { qic_off("the device refused the parameters' buffer"); return 0; }g_qic.ok = 1;- fprintf(stderr, "[VK] qwenimage chain: %d blocks on the device\n", Lr);+ fprintf(stderr, "[VK] qwenimage chain: %d blocks on the device\n", Lr); /* resident or streamed */return 1;}@@ -154,9 +239,7 @@ static int qic_attn(const VkcEncAttn *at, int N, int L) {/* One denoising step on the device: out[N][in_ch]. 0: not taken (the CPU runs it). */static int qic_forward(Dit *d, const Prefix *p, DitStep *s, const float *lat, float t, float *out) {- if (g_qic_on < 0) g_qic_on = g_vk_ready && vkc_init() &&- coli_vk_chain_decide("qwenimage", -1, COLI_VK_CHAIN_ON) == COLI_VK_CHAIN_ON;- if (!g_qic_on || g_qic.failed || vkc_lost()) return 0;+ if (g_qic_on <= 0 || g_qic.failed || vkc_lost()) return 0;if (!g_qic.ok && !qic_setup(d)) return 0;int N = s->N, L = p->L, D = d->dim, H = d->heads, hd = d->hd, M = d->mlp, C = d->in_ch;if (hd > 128 || D > 4096 || hd % 2) { qic_off("a head or a row its ops do not take"); return 0; }@@ -196,6 +279,7 @@ static int qic_forward(Dit *d, const Prefix *p, DitStep *s, const float *lat, flvkc_matmul(qic_tensor(&d->img_in), g_qic.lat, 0, g_qic.x, 0, N);for (int l = 0; ok && l < d->layers; l++) {DitBlock *B = &d->B[l];+ if (l >= g_qic.resident && !qic_stream(d, l)) { ok = 0; break; }VkcNorm nq = {N * H, hd, H, 0, D, hd, 0, D, hd, l * 2 * hd, 0, 0, d->eps, 1.f}, nk = nq;nk.x_off = nk.y_off = (int)(KO + (size_t)L * D); nk.w_off = l * 2 * hd + hd;ok = vkc_enc_norm(g_qic.x, NULL, g_qic.mod, g_qic.h, &n1) &&@@ -214,11 +298,17 @@ static int qic_forward(Dit *d, const Prefix *p, DitStep *s, const float *lat, flvkc_ew(g_qic.g, g_qic.g, g_qic.u, NULL, NULL, &sw) &&vkc_matmul(qic_tensor(&B->out), g_qic.g, 0, g_qic.h, 0, N) &&vkc_ew(g_qic.x, g_qic.x, g_qic.h, NULL, g_qic.mod, &g2) &&- vkc_submit(0) && vkc_begin(); /* a submission a block: no single one runs for seconds */+ vkc_submit(0); /* a submission a block: no single one runs for seconds */+ if (ok && l >= g_qic.resident) g_qic.ser[l & 1] = vkc_serial();+ ok = ok && vkc_begin();}ok = ok && vkc_enc_norm(g_qic.x, NULL, g_qic.mod, g_qic.h, &no) &&vkc_matmul(qic_tensor(&d->proj_out), g_qic.h, 0, g_qic.pred, 0, N) &&vkc_copy(g_qic.predd, 0, g_qic.pred, 0, (size_t)N * C) && vkc_submit(1);+ if (g_qic.streamed) { /* every frame is through: the streamed blocks' places are free */+ if (!ok && !vkc_lost()) vkc_finish();+ qic_unstream(d, 0); qic_unstream(d, 1);+ }if (!ok) {if (!vkc_lost()) { vkc_finish(); coli_vk_mark_lost_dev(0); }qic_off("a frame failed (the device was lost); this step runs again on the CPU");@@ -236,5 +326,8 @@ static void qic_report(void) {fprintf(stderr, "[VK] qwenimage chain: %llu steps, %llu frames (%llu ops, %llu matmuls, %llu tiled GEMM), ""%.1f ms waiting for the device, %.1f MiB on the device\n",g_qic.steps, st.frames, st.ops, st.matmuls, st.gemms, st.wait_ms, st.dev_bytes / 1048576.0);+ if (g_qic.streamed)+ fprintf(stderr, "[VK] qwenimage chain: %d blocks streamed, %.1f MiB uploaded\n", g_qic.streamed,+ g_qic.stream_bytes / 1048576.0);vkc_prof_print();}diff --git a/c/tests/vulkan_engines.sh b/c/tests/vulkan_engines.shindex 93c15093..a2cd7ec3 100755--- a/c/tests/vulkan_engines.sh+++ b/c/tests/vulkan_engines.sh@@ -913,6 +913,11 @@ QIPYdoneqi_chain "qwenimage chain, the attention in slices" 32 COLI_VK_ATTN_SLICE=60000grep -qa 'qwenimage chain: [0-9]* steps' qi-vk.log || { cat qi-vk.log; fail "qwenimage chain, slices: no step on the device"; }+ # a device that holds one block of the two, or none: the others go up each step+ for r in 1 0; do+ qi_chain "qwenimage chain, $r of 2 blocks resident" 8 COLI_VK_QI_RESIDENT=$r+ grep -qa "qwenimage chain: $((2 - r)) blocks streamed" qi-vk.log || { cat qi-vk.log; fail "qwenimage chain, $r resident: no block streamed"; }+ doneqi_chain "qwenimage chain, the device lost" 32 COLI_VK_CHAIN_FAULT=9grep -qa 'qwenimage chain: a frame failed' qi-vk.log || { cat qi-vk.log; fail "qwenimage chain, device lost: no loss handled"; }COLI_VULKAN=1 COLI_VK_CHAIN=1 COLI_IMG_BITS=8 COLI_IMG_ACT8=0 ./qwenimage --model qwenimage_tiny --prompt "a red fox in the snow" \@@ -1263,7 +1268,7 @@ family_kimi_mimo_sanitize() {# qwenimage's DiT on the chain: the oracle, then the attention in slices$PY tools/make_qwenimage_tiny.py qwenimage_tinylocal qa- for qa in COLI_VK_ATTN_SLICE=0 COLI_VK_ATTN_SLICE=60000; do+ for qa in COLI_VK_ATTN_SLICE=0 COLI_VK_ATTN_SLICE=60000 COLI_VK_QI_RESIDENT=0; doenv ASAN_OPTIONS=detect_leaks=0:detect_stack_use_after_return=0 UBSAN_OPTIONS=print_stacktrace=1 OMP_NUM_THREADS=2 \COLI_VULKAN=1 COLI_VK_CHAIN=1 COLI_IMG_BITS=32 COLI_IMG_ACT8=0 $qa ./qwenimage --model qwenimage_tiny --ref qwenimage_tiny/ref \> san.log 2>&1 || { cat san.log; fail "asan qwenimage chain $qa: the oracle"; }diff --git a/docs/qwen-image.md b/docs/qwen-image.mdindex 7a9b5deb..20256f08 100644--- a/docs/qwen-image.md+++ b/docs/qwen-image.md@@ -278,6 +278,16 @@ convolutions as a band's taps and one GEMM, the norms, the shortcuts; the mid blattention over every pixel on the CPU), with the CPU's decode as the fallback. Thetext encoder runs on the CPU.+A card whose free memory does not hold the transformer (it is 7.1 GB in int8: a 6 or+8 GB card) keeps the blocks that fit and uploads the others at every step, two at a+time, each going up while the block before it runs; their matrices never go through+the one-by-one path, so the device is not asked for more memory than it has. The+start says how many blocks stay (`[VK] qwenimage: 9 of 32 blocks stay on the+device`), and `COLI_VK_QI_RESIDENT=n` forces the number. On the 780M, streaming+every block costs 0.3 s on a 512x512 step (5.64 s against 5.35); a discrete card+moves them over PCIe, about 6.5 GB a step. With blocks streamed, the VAE decodes on+the CPU.+Measured on a Radeon 780M (an integrated GPU sharing the RAM of the 8-core Zen 4above), 8 steps: