Skip to content

Run frontier MoE models on hardware you already own — pure C, zero deps, experts streamed from disk. Tiny engine, immense model. 🐦

C

qwen-image: int8 x int8 GEMM with VNNI, opt-in (COLI_IMG_ACT8=1)

JustVuggc295dd421f78
diff --git a/c/qi_gemm.h b/c/qi_gemm.h
index 9d86bcc5..e2745eb6 100644
--- a/c/qi_gemm.h
+++ b/c/qi_gemm.h
@@ -157,6 +157,125 @@ static inline void qi_gemm(float *Y, const float *X, int M, const QiMat *W, cons
qi_gemm_ld(Y, W->N, X, W->K, M, W, bias);
}
+/* ---- int8 x int8 (opt-in): the activations quantized per row too ---------
+ *
+ * With VNNI one vpdpbusd does four multiply-adds per 32-bit lane where an FMA
+ * does one, so a DiT step is compute-bound on a quarter of the instructions.
+ * The price is an 8-bit activation (one scale per row, amax/127): that is a
+ * quality question, measured against the oracle, not assumed. vpdpbusd wants
+ * unsigned x signed bytes, so x is stored as q+128 and 128*sum(w) comes off the
+ * int32 sum at the end. The int32 sum over the whole K cannot overflow:
+ * K * 255 * 127 stays under 2^31 up to K = 66000.
+ *
+ * Only for QI_I8 weights and only where VNNI exists (AVX-VNNI, or AVX-512 VNNI
+ * with VL); qi_gemm_act8() falls back to qi_gemm() otherwise. */
+#if (defined(__AVXVNNI__) || (defined(__AVX512VNNI__) && defined(__AVX512VL__))) && defined(__AVX2__)
+#define QI_HAVE_VNNI 1
+#if defined(__AVX512VNNI__) && defined(__AVX512VL__)
+#define QI_DPBUSD(acc, a, b) _mm256_dpbusd_epi32(acc, a, b)
+#else
+#define QI_DPBUSD(acc, a, b) _mm256_dpbusd_avx_epi32(acc, a, b)
+#endif
+
+/* W rows [n0, n0+nr), all K (padded to Kp, a multiple of 4), as [Kp/4][16][4]
+ * bytes, plus the sum of each row for the +128 correction. */
+static void qi_pack_w8(int8_t *panel, int32_t *wsum, const QiMat *W, int n0, int nr, int Kp){
+ const int K = W->K, ld = W->ld ? W->ld : K;
+ for (int j = 0; j < QI_NR; j++) {
+ int32_t s = 0;
+ const int8_t *r = j < nr ? (const int8_t *)W->w + (int64_t)(n0 + j) * ld : NULL;
+ for (int k = 0; k < Kp; k++) {
+ int8_t v = r && k < K ? r[k] : 0;
+ panel[(k >> 2) * (QI_NR * 4) + j * 4 + (k & 3)] = v;
+ s += v;
+ }
+ wsum[j] = s;
+ }
+}
+
+static void qi_kernel8(float *Y, int ldy, const uint8_t *X, int Kp, const int8_t *panel,
+ const int32_t *wsum, const float *sx, const float *sw, int mr, int nr, const float *bias){
+ __m256i c[QI_MR][2];
+ for (int i = 0; i < QI_MR; i++) c[i][0] = c[i][1] = _mm256_setzero_si256();
+ const int kq = Kp >> 2;
+ if (mr == QI_MR) {
+ for (int q = 0; q < kq; q++) {
+ __m256i b0 = _mm256_loadu_si256((const __m256i *)(panel + q * 64));
+ __m256i b1 = _mm256_loadu_si256((const __m256i *)(panel + q * 64 + 32));
+ for (int i = 0; i < QI_MR; i++) {
+ int32_t a4; memcpy(&a4, X + (int64_t)i * Kp + q * 4, 4);
+ __m256i a = _mm256_set1_epi32(a4);
+ c[i][0] = QI_DPBUSD(c[i][0], a, b0);
+ c[i][1] = QI_DPBUSD(c[i][1], a, b1);
+ }
+ }
+ } else {
+ for (int q = 0; q < kq; q++) {
+ __m256i b0 = _mm256_loadu_si256((const __m256i *)(panel + q * 64));
+ __m256i b1 = _mm256_loadu_si256((const __m256i *)(panel + q * 64 + 32));
+ for (int i = 0; i < mr; i++) {
+ int32_t a4; memcpy(&a4, X + (int64_t)i * Kp + q * 4, 4);
+ __m256i a = _mm256_set1_epi32(a4);
+ c[i][0] = QI_DPBUSD(c[i][0], a, b0);
+ c[i][1] = QI_DPBUSD(c[i][1], a, b1);
+ }
+ }
+ }
+ int32_t t[QI_NR];
+ for (int i = 0; i < mr; i++) {
+ _mm256_storeu_si256((__m256i *)t, c[i][0]);
+ _mm256_storeu_si256((__m256i *)(t + 8), c[i][1]);
+ for (int j = 0; j < nr; j++)
+ Y[(int64_t)i * ldy + j] = (float)(t[j] - 128 * wsum[j]) * sx[i] * sw[j] + (bias ? bias[j] : 0.f);
+ }
+}
+#endif
+
+/* Y = X . W^T with X quantized to int8 per row. W must be QI_I8. */
+static void qi_gemm_act8(float *Y, int ldy, const float *X, int ldx, int M, const QiMat *W, const float *bias){
+#ifdef QI_HAVE_VNNI
+ const int N = W->N, K = W->K, Kp = (K + 3) & ~3;
+ if (W->fmt != QI_I8 || M <= 0 || N <= 0) { qi_gemm_ld(Y, ldy, X, ldx, M, W, bias); return; }
+ uint8_t *xq = (uint8_t *)malloc((size_t)M * Kp);
+ float *sx = (float *)malloc(sizeof(float) * (size_t)M);
+ if (!xq || !sx) { fprintf(stderr, "OOM qi_gemm_act8\n"); exit(1); }
+ #pragma omp parallel for schedule(static)
+ for (int m = 0; m < M; m++) {
+ const float *r = X + (int64_t)m * ldx;
+ float am = 0.f;
+ for (int k = 0; k < K; k++) { float a = r[k] < 0 ? -r[k] : r[k]; if (a > am) am = a; }
+ float s = am > 1e-30f ? am / 127.f : 1.f, inv = 1.f / s;
+ sx[m] = s;
+ uint8_t *d = xq + (int64_t)m * Kp;
+ for (int k = 0; k < K; k++) { float v = r[k] * inv; int iv = (int)(v < 0 ? v - 0.5f : v + 0.5f); d[k] = (uint8_t)(iv + 128); }
+ for (int k = K; k < Kp; k++) d[k] = 128;
+ }
+ const int nblocks = (N + QI_NR - 1) / QI_NR, mblocks = (M + QI_MC - 1) / QI_MC;
+ #pragma omp parallel
+ {
+ int8_t *panel = (int8_t *)aligned_alloc(64, (size_t)Kp * QI_NR);
+ int32_t wsum[QI_NR];
+ if (!panel) { fprintf(stderr, "OOM qi_gemm_act8 panel\n"); exit(1); }
+ #pragma omp for schedule(dynamic, 1) collapse(2)
+ for (int nb = 0; nb < nblocks; nb++)
+ for (int mb = 0; mb < mblocks; mb++) {
+ int n0 = nb * QI_NR, nr = N - n0 < QI_NR ? N - n0 : QI_NR;
+ int m0 = mb * QI_MC, mc = M - m0 < QI_MC ? M - m0 : QI_MC;
+ qi_pack_w8(panel, wsum, W, n0, nr, Kp);
+ for (int i = 0; i < mc; i += QI_MR) {
+ int mr = mc - i < QI_MR ? mc - i : QI_MR;
+ qi_kernel8(Y + (int64_t)(m0 + i) * ldy + n0, ldy, xq + (int64_t)(m0 + i) * Kp, Kp,
+ panel, wsum, sx + m0 + i, W->sc + n0, mr, nr, bias ? bias + n0 : NULL);
+ }
+ }
+ free(panel);
+ }
+ free(xq); free(sx);
+#else
+ qi_gemm_ld(Y, ldy, X, ldx, M, W, bias);
+#endif
+}
+
/* Per-row int8 quantization of an f32 matrix (max-abs / 127, round to nearest). */
static void qi_quantize_i8(const float *src, int N, int K, int8_t *q, float *sc){
#pragma omp parallel for schedule(static)
diff --git a/c/qwenimage.c b/c/qwenimage.c
index f958299e..89154ba6 100644
--- a/c/qwenimage.c
+++ b/c/qwenimage.c
@@ -90,6 +90,7 @@ static int jbool(jval *o, const char *k, int def){
/* ---- weights ------------------------------------------------------------ */
static int g_bits = 8; /* COLI_IMG_BITS: storage of the big matrices */
+static int g_act8 = 0; /* COLI_IMG_ACT8=1: int8 activations on the int8 matrices (VNNI) */
typedef struct { QiMat m; void *own; float *own_sc; } Lin;
@@ -142,7 +143,10 @@ static float *vec_load(shards *S, const char *name, int n){
st_read_f32_cap(S, name, v, n, 1);
return v;
}
-static inline void linear(float *y, const float *x, int M, const Lin *l){ qi_gemm(y, x, M, &l->m, NULL); }
+static inline void linear(float *y, const float *x, int M, const Lin *l){
+ if (g_act8 && l->m.fmt == QI_I8) qi_gemm_act8(y, l->m.N, x, l->m.K, M, &l->m, NULL);
+ else qi_gemm(y, x, M, &l->m, NULL);
+}
/* ---- small kernels -------------------------------------------------------- */
@@ -1123,6 +1127,7 @@ static void usage(void){
" qwenimage --model DIR --serve\n"
" qwenimage --model DIR --ref REFDIR\n"
"env: COLI_IMG_BITS=8|16|32 weight storage (default 8: int8 rows)\n"
+ " COLI_IMG_ACT8=1 int8 activations on the int8 matrices (VNNI, about 2x; quality measured separately)\n"
" COLI_IMG_TE=resident|stage keep the text encoder loaded between prompts (serve default: resident)\n");
}
@@ -1152,6 +1157,11 @@ int main(int argc, char **argv){
if (!model || (!serve && !ref && !getenv("QWENIMAGE_PRINT_TOKENS") && (!prompt || !out))) { usage(); return 2; }
const char *b = getenv("COLI_IMG_BITS");
if (b && *b) { g_bits = atoi(b); if (g_bits != 8 && g_bits != 16 && g_bits != 32) { usage(); return 2; } }
+ const char *a8 = getenv("COLI_IMG_ACT8");
+ g_act8 = a8 && *a8 == '1';
+#ifndef QI_HAVE_VNNI
+ if (g_act8) { fprintf(stderr, "[qwenimage] COLI_IMG_ACT8 needs VNNI; this build has none, staying on f32 activations\n"); g_act8 = 0; }
+#endif
if (!seed_set && !serve && !ref) seed = (uint64_t)time(NULL);
static Engine e;
engine_init(&e, model);
diff --git a/c/tests/test_qi_gemm.c b/c/tests/test_qi_gemm.c
index 7a9b54ca..76ae1aff 100644
--- a/c/tests/test_qi_gemm.c
+++ b/c/tests/test_qi_gemm.c
@@ -32,6 +32,39 @@ static int check(int M, int N, int K, int fmt, int use_bias){
return ok;
}
+
+static int check_act8(int M, int N, int K){
+#ifdef QI_HAVE_VNNI
+ uint32_t s = 99u + M + N * 3 + K * 5;
+ float *X = malloc(sizeof(float) * M * K), *Wf = malloc(sizeof(float) * N * K), *Y = malloc(sizeof(float) * M * N);
+ float *b = malloc(sizeof(float) * N); int8_t *q = malloc((size_t)N * K); float *sc = malloc(sizeof(float) * N);
+ for (int i = 0; i < M * K; i++) X[i] = frand(&s) * (i % 7 == 0 ? 4.f : 1.f);
+ for (int i = 0; i < N * K; i++) Wf[i] = frand(&s);
+ for (int i = 0; i < N; i++) b[i] = frand(&s);
+ qi_quantize_i8(Wf, N, K, q, sc);
+ QiMat W = { QI_I8, N, K, q, sc, 0 };
+ qi_gemm_act8(Y, N, X, K, M, &W, b);
+ double maxerr = 0;
+ for (int m = 0; m < M; m++) {
+ float am = 0; for (int k = 0; k < K; k++) { float a = fabsf(X[(int64_t)m*K+k]); if (a > am) am = a; }
+ float sx = am / 127.f, inv = 1.f / sx;
+ for (int n = 0; n < N; n++) {
+ long acc = 0;
+ for (int k = 0; k < K; k++) { float v = X[(int64_t)m*K+k] * inv; int iv = (int)(v < 0 ? v - 0.5f : v + 0.5f); acc += (long)iv * q[(int64_t)n*K+k]; }
+ double want = (double)acc * sx * sc[n] + b[n];
+ double e = fabs(want - Y[(int64_t)m*N+n]) / (1.0 + fabs(want));
+ if (e > maxerr) maxerr = e;
+ }
+ }
+ int ok = maxerr < 1e-5;
+ printf("%s act8 M=%d N=%d K=%d maxrel=%.2e\n", ok ? "ok " : "FAIL", M, N, K, maxerr);
+ free(X); free(Wf); free(Y); free(b); free(q); free(sc);
+ return ok;
+#else
+ printf("skip act8 (no VNNI)\n"); return 1;
+#endif
+}
+
int main(int argc, char **argv){
if (argc > 1 && !strcmp(argv[1], "bench")) {
int M = argc > 2 ? atoi(argv[2]) : 1296, N = argc > 3 ? atoi(argv[3]) : 12288, K = argc > 4 ? atoi(argv[4]) : 4096;
@@ -40,18 +73,21 @@ int main(int argc, char **argv){
uint32_t s = 7; for (size_t i = 0; i < (size_t)M * K; i++) X[i] = frand(&s);
for (size_t i = 0; i < (size_t)N * K; i++) q[i] = (int8_t)(frand(&s) * 127); for (int i = 0; i < N; i++) sc[i] = 0.01f;
QiMat W = { QI_I8, N, K, q, sc, 0 };
- qi_gemm(Y, X, M, &W, NULL);
+ int act8 = argc > 5 && !strcmp(argv[5], "act8");
+ if (act8) qi_gemm_act8(Y, N, X, K, M, &W, NULL); else qi_gemm(Y, X, M, &W, NULL);
struct timespec a, b; clock_gettime(CLOCK_MONOTONIC, &a);
- int reps = 3; for (int r = 0; r < reps; r++) qi_gemm(Y, X, M, &W, NULL);
+ int reps = 3; for (int r = 0; r < reps; r++) { if (act8) qi_gemm_act8(Y, N, X, K, M, &W, NULL); else qi_gemm(Y, X, M, &W, NULL); }
clock_gettime(CLOCK_MONOTONIC, &b);
double t = (b.tv_sec - a.tv_sec) + (b.tv_nsec - a.tv_nsec) * 1e-9;
- printf("M=%d N=%d K=%d int8: %.1f ms, %.1f GFLOP/s\n", M, N, K, t / reps * 1e3, 2.0 * M * N * K * reps / t * 1e-9);
+ printf("M=%d N=%d K=%d %s: %.1f ms, %.1f GFLOP/s\n", M, N, K, act8 ? "int8 x int8 (VNNI)" : "int8 weights, f32 act", t / reps * 1e3, 2.0 * M * N * K * reps / t * 1e-9);
return 0;
}
int ok = 1;
int shapes[][3] = { {1,1,1}, {5,17,33}, {6,16,256}, {7,31,257}, {64,48,300}, {193,40,520}, {13,100,64} };
for (unsigned i = 0; i < sizeof shapes / sizeof shapes[0]; i++)
for (int fmt = 0; fmt < 3; fmt++) ok &= check(shapes[i][0], shapes[i][1], shapes[i][2], fmt, i & 1);
+ int act_shapes[][3] = { {1,1,1}, {7,31,257}, {6,16,256}, {193,40,522}, {13,100,64} };
+ for (unsigned i = 0; i < sizeof act_shapes / sizeof act_shapes[0]; i++) ok &= check_act8(act_shapes[i][0], act_shapes[i][1], act_shapes[i][2]);
printf(ok ? "qi_gemm: all ok\n" : "qi_gemm: FAILED\n");
return !ok;
}