Run frontier MoE models on hardware you already own — pure C, zero deps, experts streamed from disk. Tiny engine, immense model. 🐦
C
qwen-image vae: 3x row-tap im2col instead of the 9x one
JustVugg3f517b663ebc
diff --git a/c/qwenimage_vae.h b/c/qwenimage_vae.hindex 16bde7e4..0c7e71de 100644--- a/c/qwenimage_vae.h+++ b/c/qwenimage_vae.h@@ -15,10 +15,10 @@* + SiLU, conv_out, clamp to [-1, 1], then VaeImageProcessor.postprocess* ((x*0.5 + 0.5).clamp(0, 1) * 255, numpy round-half-even, uint8).*- * Activations are channel-last [H][W][C]. A 3x3 conv is im2col times the weight- * through qi_gemm; the weight is permuted at load to [Cout][ky][kx][Cin] so an- * im2col row is nine contiguous copies of Cin floats. The im2col only exists for- * a band of output rows (qiv_col_floats), and the input rows a band has consumed+ * Activations are channel-last [H][W][C]. A 3x3 conv is one qi_gemm per band of+ * output rows over a 3x (not 9x) im2col, see qiv_conv3; the weight is permuted+ * at load to [Cout][kx][ky][Cin] to match. The im2col only exists for a band of+ * output rows (qiv_col_floats), and the input rows a band has consumed* are handed back to the OS when nothing reads them later (qiv_release): at* 1024x1024 the last up blocks hold 288-channel maps of 1.2 GB each, and without* that the decode would not fit next to the rest of the pipeline.@@ -56,7 +56,7 @@ static size_t qiv_attn_floats = (size_t)4 << 20;typedef struct {int cin, cout, k; /* k = 1 or 3 */- float *w, *b; /* w: [cout][ky][kx][cin] */+ float *w, *b; /* w: [cout][kx][ky][cin] */QiMat m;} QivConv;@@ -146,13 +146,14 @@ static int qiv_load_conv(shards *S, QiVae *v, const char *pfx, int cin, int coutfloat *w = qiv_read(S, nm, (int64_t)cout * cin * k * k, err, errn);if (!w) return -1;if (k > 1) {- /* [cout][cin][ky][kx] -> [cout][ky][kx][cin] */+ /* [cout][cin][ky][kx] -> [cout][kx][ky][cin], the order qiv_conv3 reads taps in */float *p = qiv_alloc((size_t)cout * cin * k * k);if (!p) { free(w); snprintf(err, errn, "out of memory"); return -1; }for (int o = 0; o < cout; o++)for (int i = 0; i < cin; i++)- for (int t = 0; t < k * k; t++)- p[((size_t)o * k * k + t) * cin + i] = w[((size_t)o * cin + i) * k * k + t];+ for (int ky = 0; ky < k; ky++)+ for (int kx = 0; kx < k; kx++)+ p[((size_t)o * k * k + kx * k + ky) * cin + i] = w[((size_t)o * cin + i) * k * k + ky * k + kx];free(w); w = p;}snprintf(nm, sizeof nm, "%s.bias", pfx);@@ -360,28 +361,35 @@ typedef struct {/* y[Ho][Wo][cout] = conv3x3(pre(x)) (+ epilogue), with pre = RMS norm + SiLU if* gamma, and x nearest-upsampled 2x first if up. Processed in bands of output- * rows: prepared input rows (with halo and zero padding) -> im2col -> qi_gemm. */+ * rows. P holds the band's input rows after pre, with a one-row halo and a zero+ * column on each side (width Wp = Wo + 2). Q interleaves three consecutive P+ * rows: Q[r][u][ky] = P[r + ky][u]. Then the nine taps of output pixel (r, u)+ * are the 9*C contiguous floats at Q[r][u], ordered [kx][ky][c], and the next+ * pixel starts 3*C floats later: a GEMM with ldx = 3*C over the rows r*Wp + u+ * reads them in place. Q costs 3 copies of the input instead of the 9 of a full+ * im2col; the two rows per band that straddle a line end are computed and+ * dropped. */static int qiv_conv3(const QivConv *cv, float *x, int H, int W, const float *gamma, int up,float *y, const QivEpi *ep){- const int C = cv->cin, Co = cv->cout, K = 9 * C;+ const int C = cv->cin, Co = cv->cout, C3 = 3 * C;const int Ho = H << up, Wo = W << up, Wp = Wo + 2;const float scale = (float)sqrt((double)C);- size_t rowcol = (size_t)Wo * K;- int band = (int)(qiv_col_floats / rowcol);+ const size_t rowq = (size_t)Wp * C3;+ int band = (int)(qiv_col_floats / rowq);if (band < 1) band = 1;if (band > Ho) band = Ho;float *P = qiv_alloc((size_t)(band + 2) * Wp * C);- float *col = qiv_alloc((size_t)band * rowcol);- float *T = qiv_alloc((size_t)band * Wo * Co);+ float *Q = qiv_alloc((size_t)band * rowq);+ float *T = qiv_alloc((size_t)band * Wp * Co);float *T2 = ep && ep->sc ? qiv_alloc((size_t)band * Wo * Co) : NULL;- if (!P || !col || !T || (ep && ep->sc && !T2)) { free(P); free(col); free(T); free(T2); return -1; }+ if (!P || !Q || !T || (ep && ep->sc && !T2)) { free(P); free(Q); free(T); free(T2); return -1; }const size_t xtot = (size_t)H * W * C;const int scC = ep && ep->sc ? ep->sc->cin : 0;size_t xrel = 0, screl = 0, duprel = 0;for (int y0 = 0; y0 < Ho; y0 += band) {const int nb = Ho - y0 < band ? Ho - y0 : band;- /* prepared rows r = 0..nb+1 hold virtual row y0-1+r, one zero column each side */+ /* P row r = virtual input row y0-1+r */#pragma omp parallel for collapse(2) schedule(static)for (int r = 0; r < nb + 2; r++)for (int u = 0; u < Wp; u++) {@@ -394,28 +402,29 @@ static int qiv_conv3(const QivConv *cv, float *x, int H, int W, const float *gam}#pragma omp parallel for collapse(2) schedule(static)for (int r = 0; r < nb; r++)- for (int u = 0; u < Wo; u++) {- float *d = col + ((size_t)r * Wo + u) * K;+ for (int u = 0; u < Wp; u++) {+ float *d = Q + ((size_t)r * Wp + u) * C3;for (int ky = 0; ky < 3; ky++)- memcpy(d + ky * 3 * C, P + ((size_t)(r + ky) * Wp + u) * C, sizeof(float) * 3 * C);+ memcpy(d + ky * C, P + ((size_t)(r + ky) * Wp + u) * C, sizeof(float) * C);}- const int M = nb * Wo;- qi_gemm(T, col, M, &cv->m, cv->b);- if (T2) qi_gemm(T2, ep->scx + (size_t)y0 * Wo * scC, M, &ep->sc->m, ep->sc->b);- #pragma omp parallel for schedule(static)- for (int p = 0; p < M; p++) {- float *t = T + (size_t)p * Co;- size_t gp = (size_t)y0 * Wo + p;- if (T2) { const float *t2 = T2 + (size_t)p * Co; for (int o = 0; o < Co; o++) t[o] += t2[o]; }- if (ep && ep->dup) {- int Y = y0 + p / Wo, X = p % Wo;- const float *s = ep->dup + ((size_t)(Y >> 1) * (Wo >> 1) + (X >> 1)) * ep->dupC;- const int *tab = ep->dupt + ((Y & 1) * 2 + (X & 1));- for (int o = 0; o < Co; o++) t[o] += s[tab[o * 4]];+ qi_gemm_ld(T, Co, Q, C3, nb * Wp - 2, &cv->m, cv->b);+ if (T2) qi_gemm(T2, ep->scx + (size_t)y0 * Wo * scC, nb * Wo, &ep->sc->m, ep->sc->b);+ #pragma omp parallel for collapse(2) schedule(static)+ for (int r = 0; r < nb; r++)+ for (int u = 0; u < Wo; u++) {+ const int p = r * Wo + u;+ float *t = T + ((size_t)r * Wp + u) * Co;+ size_t gp = (size_t)y0 * Wo + p;+ if (T2) { const float *t2 = T2 + (size_t)p * Co; for (int o = 0; o < Co; o++) t[o] += t2[o]; }+ if (ep && ep->dup) {+ int Y = y0 + r;+ const float *s = ep->dup + ((size_t)(Y >> 1) * (Wo >> 1) + (u >> 1)) * ep->dupC;+ const int *tab = ep->dupt + ((Y & 1) * 2 + (u & 1));+ for (int o = 0; o < Co; o++) t[o] += s[tab[o * 4]];+ }+ if (ep && ep->res) { const float *rr = ep->res + gp * Co; for (int o = 0; o < Co; o++) t[o] += rr[o]; }+ memcpy(y + gp * Co, t, sizeof(float) * Co);}- if (ep && ep->res) { const float *rr = ep->res + gp * Co; for (int o = 0; o < Co; o++) t[o] += rr[o]; }- memcpy(y + gp * Co, t, sizeof(float) * Co);- }const int y1 = y0 + nb;if (ep && ep->rel_x) {size_t upto = y1 >= Ho ? xtot : (size_t)((y1 - 1) >> up) * W * C;@@ -431,7 +440,7 @@ static int qiv_conv3(const QivConv *cv, float *x, int H, int W, const float *gamqiv_release((float *)ep->dup, tot, duprel, upto); duprel = upto;}}- free(P); free(col); free(T); free(T2);+ free(P); free(Q); free(T); free(T2);return 0;}