Run frontier MoE models on hardware you already own — pure C, zero deps, experts streamed from disk. Tiny engine, immense model. 🐦
C
qwen-image: int8 x int8 GEMM with VNNI, opt-in (COLI_IMG_ACT8=1)
JustVuggc295dd421f78
diff --git a/c/qi_gemm.h b/c/qi_gemm.hindex 9d86bcc5..e2745eb6 100644--- a/c/qi_gemm.h+++ b/c/qi_gemm.h@@ -157,6 +157,125 @@ static inline void qi_gemm(float *Y, const float *X, int M, const QiMat *W, consqi_gemm_ld(Y, W->N, X, W->K, M, W, bias);}+/* ---- int8 x int8 (opt-in): the activations quantized per row too ---------+ *+ * With VNNI one vpdpbusd does four multiply-adds per 32-bit lane where an FMA+ * does one, so a DiT step is compute-bound on a quarter of the instructions.+ * The price is an 8-bit activation (one scale per row, amax/127): that is a+ * quality question, measured against the oracle, not assumed. vpdpbusd wants+ * unsigned x signed bytes, so x is stored as q+128 and 128*sum(w) comes off the+ * int32 sum at the end. The int32 sum over the whole K cannot overflow:+ * K * 255 * 127 stays under 2^31 up to K = 66000.+ *+ * Only for QI_I8 weights and only where VNNI exists (AVX-VNNI, or AVX-512 VNNI+ * with VL); qi_gemm_act8() falls back to qi_gemm() otherwise. */+#if (defined(__AVXVNNI__) || (defined(__AVX512VNNI__) && defined(__AVX512VL__))) && defined(__AVX2__)+#define QI_HAVE_VNNI 1+#if defined(__AVX512VNNI__) && defined(__AVX512VL__)+#define QI_DPBUSD(acc, a, b) _mm256_dpbusd_epi32(acc, a, b)+#else+#define QI_DPBUSD(acc, a, b) _mm256_dpbusd_avx_epi32(acc, a, b)+#endif++/* W rows [n0, n0+nr), all K (padded to Kp, a multiple of 4), as [Kp/4][16][4]+ * bytes, plus the sum of each row for the +128 correction. */+static void qi_pack_w8(int8_t *panel, int32_t *wsum, const QiMat *W, int n0, int nr, int Kp){+ const int K = W->K, ld = W->ld ? W->ld : K;+ for (int j = 0; j < QI_NR; j++) {+ int32_t s = 0;+ const int8_t *r = j < nr ? (const int8_t *)W->w + (int64_t)(n0 + j) * ld : NULL;+ for (int k = 0; k < Kp; k++) {+ int8_t v = r && k < K ? r[k] : 0;+ panel[(k >> 2) * (QI_NR * 4) + j * 4 + (k & 3)] = v;+ s += v;+ }+ wsum[j] = s;+ }+}++static void qi_kernel8(float *Y, int ldy, const uint8_t *X, int Kp, const int8_t *panel,+ const int32_t *wsum, const float *sx, const float *sw, int mr, int nr, const float *bias){+ __m256i c[QI_MR][2];+ for (int i = 0; i < QI_MR; i++) c[i][0] = c[i][1] = _mm256_setzero_si256();+ const int kq = Kp >> 2;+ if (mr == QI_MR) {+ for (int q = 0; q < kq; q++) {+ __m256i b0 = _mm256_loadu_si256((const __m256i *)(panel + q * 64));+ __m256i b1 = _mm256_loadu_si256((const __m256i *)(panel + q * 64 + 32));+ for (int i = 0; i < QI_MR; i++) {+ int32_t a4; memcpy(&a4, X + (int64_t)i * Kp + q * 4, 4);+ __m256i a = _mm256_set1_epi32(a4);+ c[i][0] = QI_DPBUSD(c[i][0], a, b0);+ c[i][1] = QI_DPBUSD(c[i][1], a, b1);+ }+ }+ } else {+ for (int q = 0; q < kq; q++) {+ __m256i b0 = _mm256_loadu_si256((const __m256i *)(panel + q * 64));+ __m256i b1 = _mm256_loadu_si256((const __m256i *)(panel + q * 64 + 32));+ for (int i = 0; i < mr; i++) {+ int32_t a4; memcpy(&a4, X + (int64_t)i * Kp + q * 4, 4);+ __m256i a = _mm256_set1_epi32(a4);+ c[i][0] = QI_DPBUSD(c[i][0], a, b0);+ c[i][1] = QI_DPBUSD(c[i][1], a, b1);+ }+ }+ }+ int32_t t[QI_NR];+ for (int i = 0; i < mr; i++) {+ _mm256_storeu_si256((__m256i *)t, c[i][0]);+ _mm256_storeu_si256((__m256i *)(t + 8), c[i][1]);+ for (int j = 0; j < nr; j++)+ Y[(int64_t)i * ldy + j] = (float)(t[j] - 128 * wsum[j]) * sx[i] * sw[j] + (bias ? bias[j] : 0.f);+ }+}+#endif++/* Y = X . W^T with X quantized to int8 per row. W must be QI_I8. */+static void qi_gemm_act8(float *Y, int ldy, const float *X, int ldx, int M, const QiMat *W, const float *bias){+#ifdef QI_HAVE_VNNI+ const int N = W->N, K = W->K, Kp = (K + 3) & ~3;+ if (W->fmt != QI_I8 || M <= 0 || N <= 0) { qi_gemm_ld(Y, ldy, X, ldx, M, W, bias); return; }+ uint8_t *xq = (uint8_t *)malloc((size_t)M * Kp);+ float *sx = (float *)malloc(sizeof(float) * (size_t)M);+ if (!xq || !sx) { fprintf(stderr, "OOM qi_gemm_act8\n"); exit(1); }+ #pragma omp parallel for schedule(static)+ for (int m = 0; m < M; m++) {+ const float *r = X + (int64_t)m * ldx;+ float am = 0.f;+ for (int k = 0; k < K; k++) { float a = r[k] < 0 ? -r[k] : r[k]; if (a > am) am = a; }+ float s = am > 1e-30f ? am / 127.f : 1.f, inv = 1.f / s;+ sx[m] = s;+ uint8_t *d = xq + (int64_t)m * Kp;+ for (int k = 0; k < K; k++) { float v = r[k] * inv; int iv = (int)(v < 0 ? v - 0.5f : v + 0.5f); d[k] = (uint8_t)(iv + 128); }+ for (int k = K; k < Kp; k++) d[k] = 128;+ }+ const int nblocks = (N + QI_NR - 1) / QI_NR, mblocks = (M + QI_MC - 1) / QI_MC;+ #pragma omp parallel+ {+ int8_t *panel = (int8_t *)aligned_alloc(64, (size_t)Kp * QI_NR);+ int32_t wsum[QI_NR];+ if (!panel) { fprintf(stderr, "OOM qi_gemm_act8 panel\n"); exit(1); }+ #pragma omp for schedule(dynamic, 1) collapse(2)+ for (int nb = 0; nb < nblocks; nb++)+ for (int mb = 0; mb < mblocks; mb++) {+ int n0 = nb * QI_NR, nr = N - n0 < QI_NR ? N - n0 : QI_NR;+ int m0 = mb * QI_MC, mc = M - m0 < QI_MC ? M - m0 : QI_MC;+ qi_pack_w8(panel, wsum, W, n0, nr, Kp);+ for (int i = 0; i < mc; i += QI_MR) {+ int mr = mc - i < QI_MR ? mc - i : QI_MR;+ qi_kernel8(Y + (int64_t)(m0 + i) * ldy + n0, ldy, xq + (int64_t)(m0 + i) * Kp, Kp,+ panel, wsum, sx + m0 + i, W->sc + n0, mr, nr, bias ? bias + n0 : NULL);+ }+ }+ free(panel);+ }+ free(xq); free(sx);+#else+ qi_gemm_ld(Y, ldy, X, ldx, M, W, bias);+#endif+}+/* Per-row int8 quantization of an f32 matrix (max-abs / 127, round to nearest). */static void qi_quantize_i8(const float *src, int N, int K, int8_t *q, float *sc){#pragma omp parallel for schedule(static)diff --git a/c/qwenimage.c b/c/qwenimage.cindex f958299e..89154ba6 100644--- a/c/qwenimage.c+++ b/c/qwenimage.c@@ -90,6 +90,7 @@ static int jbool(jval *o, const char *k, int def){/* ---- weights ------------------------------------------------------------ */static int g_bits = 8; /* COLI_IMG_BITS: storage of the big matrices */+static int g_act8 = 0; /* COLI_IMG_ACT8=1: int8 activations on the int8 matrices (VNNI) */typedef struct { QiMat m; void *own; float *own_sc; } Lin;@@ -142,7 +143,10 @@ static float *vec_load(shards *S, const char *name, int n){st_read_f32_cap(S, name, v, n, 1);return v;}-static inline void linear(float *y, const float *x, int M, const Lin *l){ qi_gemm(y, x, M, &l->m, NULL); }+static inline void linear(float *y, const float *x, int M, const Lin *l){+ if (g_act8 && l->m.fmt == QI_I8) qi_gemm_act8(y, l->m.N, x, l->m.K, M, &l->m, NULL);+ else qi_gemm(y, x, M, &l->m, NULL);+}/* ---- small kernels -------------------------------------------------------- */@@ -1123,6 +1127,7 @@ static void usage(void){" qwenimage --model DIR --serve\n"" qwenimage --model DIR --ref REFDIR\n""env: COLI_IMG_BITS=8|16|32 weight storage (default 8: int8 rows)\n"+ " COLI_IMG_ACT8=1 int8 activations on the int8 matrices (VNNI, about 2x; quality measured separately)\n"" COLI_IMG_TE=resident|stage keep the text encoder loaded between prompts (serve default: resident)\n");}@@ -1152,6 +1157,11 @@ int main(int argc, char **argv){if (!model || (!serve && !ref && !getenv("QWENIMAGE_PRINT_TOKENS") && (!prompt || !out))) { usage(); return 2; }const char *b = getenv("COLI_IMG_BITS");if (b && *b) { g_bits = atoi(b); if (g_bits != 8 && g_bits != 16 && g_bits != 32) { usage(); return 2; } }+ const char *a8 = getenv("COLI_IMG_ACT8");+ g_act8 = a8 && *a8 == '1';+#ifndef QI_HAVE_VNNI+ if (g_act8) { fprintf(stderr, "[qwenimage] COLI_IMG_ACT8 needs VNNI; this build has none, staying on f32 activations\n"); g_act8 = 0; }+#endifif (!seed_set && !serve && !ref) seed = (uint64_t)time(NULL);static Engine e;engine_init(&e, model);diff --git a/c/tests/test_qi_gemm.c b/c/tests/test_qi_gemm.cindex 7a9b54ca..76ae1aff 100644--- a/c/tests/test_qi_gemm.c+++ b/c/tests/test_qi_gemm.c@@ -32,6 +32,39 @@ static int check(int M, int N, int K, int fmt, int use_bias){return ok;}++static int check_act8(int M, int N, int K){+#ifdef QI_HAVE_VNNI+ uint32_t s = 99u + M + N * 3 + K * 5;+ float *X = malloc(sizeof(float) * M * K), *Wf = malloc(sizeof(float) * N * K), *Y = malloc(sizeof(float) * M * N);+ float *b = malloc(sizeof(float) * N); int8_t *q = malloc((size_t)N * K); float *sc = malloc(sizeof(float) * N);+ for (int i = 0; i < M * K; i++) X[i] = frand(&s) * (i % 7 == 0 ? 4.f : 1.f);+ for (int i = 0; i < N * K; i++) Wf[i] = frand(&s);+ for (int i = 0; i < N; i++) b[i] = frand(&s);+ qi_quantize_i8(Wf, N, K, q, sc);+ QiMat W = { QI_I8, N, K, q, sc, 0 };+ qi_gemm_act8(Y, N, X, K, M, &W, b);+ double maxerr = 0;+ for (int m = 0; m < M; m++) {+ float am = 0; for (int k = 0; k < K; k++) { float a = fabsf(X[(int64_t)m*K+k]); if (a > am) am = a; }+ float sx = am / 127.f, inv = 1.f / sx;+ for (int n = 0; n < N; n++) {+ long acc = 0;+ for (int k = 0; k < K; k++) { float v = X[(int64_t)m*K+k] * inv; int iv = (int)(v < 0 ? v - 0.5f : v + 0.5f); acc += (long)iv * q[(int64_t)n*K+k]; }+ double want = (double)acc * sx * sc[n] + b[n];+ double e = fabs(want - Y[(int64_t)m*N+n]) / (1.0 + fabs(want));+ if (e > maxerr) maxerr = e;+ }+ }+ int ok = maxerr < 1e-5;+ printf("%s act8 M=%d N=%d K=%d maxrel=%.2e\n", ok ? "ok " : "FAIL", M, N, K, maxerr);+ free(X); free(Wf); free(Y); free(b); free(q); free(sc);+ return ok;+#else+ printf("skip act8 (no VNNI)\n"); return 1;+#endif+}+int main(int argc, char **argv){if (argc > 1 && !strcmp(argv[1], "bench")) {int M = argc > 2 ? atoi(argv[2]) : 1296, N = argc > 3 ? atoi(argv[3]) : 12288, K = argc > 4 ? atoi(argv[4]) : 4096;@@ -40,18 +73,21 @@ int main(int argc, char **argv){uint32_t s = 7; for (size_t i = 0; i < (size_t)M * K; i++) X[i] = frand(&s);for (size_t i = 0; i < (size_t)N * K; i++) q[i] = (int8_t)(frand(&s) * 127); for (int i = 0; i < N; i++) sc[i] = 0.01f;QiMat W = { QI_I8, N, K, q, sc, 0 };- qi_gemm(Y, X, M, &W, NULL);+ int act8 = argc > 5 && !strcmp(argv[5], "act8");+ if (act8) qi_gemm_act8(Y, N, X, K, M, &W, NULL); else qi_gemm(Y, X, M, &W, NULL);struct timespec a, b; clock_gettime(CLOCK_MONOTONIC, &a);- int reps = 3; for (int r = 0; r < reps; r++) qi_gemm(Y, X, M, &W, NULL);+ int reps = 3; for (int r = 0; r < reps; r++) { if (act8) qi_gemm_act8(Y, N, X, K, M, &W, NULL); else qi_gemm(Y, X, M, &W, NULL); }clock_gettime(CLOCK_MONOTONIC, &b);double t = (b.tv_sec - a.tv_sec) + (b.tv_nsec - a.tv_nsec) * 1e-9;- printf("M=%d N=%d K=%d int8: %.1f ms, %.1f GFLOP/s\n", M, N, K, t / reps * 1e3, 2.0 * M * N * K * reps / t * 1e-9);+ printf("M=%d N=%d K=%d %s: %.1f ms, %.1f GFLOP/s\n", M, N, K, act8 ? "int8 x int8 (VNNI)" : "int8 weights, f32 act", t / reps * 1e3, 2.0 * M * N * K * reps / t * 1e-9);return 0;}int ok = 1;int shapes[][3] = { {1,1,1}, {5,17,33}, {6,16,256}, {7,31,257}, {64,48,300}, {193,40,520}, {13,100,64} };for (unsigned i = 0; i < sizeof shapes / sizeof shapes[0]; i++)for (int fmt = 0; fmt < 3; fmt++) ok &= check(shapes[i][0], shapes[i][1], shapes[i][2], fmt, i & 1);+ int act_shapes[][3] = { {1,1,1}, {7,31,257}, {6,16,256}, {193,40,522}, {13,100,64} };+ for (unsigned i = 0; i < sizeof act_shapes / sizeof act_shapes[0]; i++) ok &= check_act8(act_shapes[i][0], act_shapes[i][1], act_shapes[i][2]);printf(ok ? "qi_gemm: all ok\n" : "qi_gemm: FAILED\n");return !ok;}