Skip to content

Run frontier MoE models on hardware you already own — pure C, zero deps, experts streamed from disk. Tiny engine, immense model. 🐦

C
15.1 KB
Raw Blame
/* qi_gemm.h -- the one GEMM of the image engine.
 *
 * Y[M][N] = X[M][K] . W[N][K]^T (+ bias[N]), X and Y f32 row-major, W row-major
 * in one of three storage formats: f32, bf16, or int8 with one f32 scale per
 * row. Every linear layer of the text encoder and the DiT, every convolution
 * of the VAE (through im2col) and both products of the attention go through
 * here, so this is where the time of an image goes.
 *
 * Why not matmul_q_batch: the text engines' kernels are GEMVs that stream the
 * weight row once per output and sweep every activation row under it. That is
 * right for decode (one or two rows) and wrong for a DiT step, where M is the
 * number of image tokens (1296 at 768x432): the activations stop fitting in
 * cache and each output row re-reads all of them. Here W is dequantized into
 * f32 panels of NR rows x KC columns, X is walked in blocks of MC rows that stay
 * in L2, and a register tile of MR x NR accumulates in FMA registers.
 *
 * The weights stay int8 or bf16 in memory; the f32 copy only ever exists as one
 * packed panel per thread. Activations stay f32: no activation quantization, so
 * the result is the f32 product of the dequantized weights up to summation
 * order. */
#ifndef COLIBRI_QI_GEMM_H
#define COLIBRI_QI_GEMM_H
#include <stdint.h>
#include <stdlib.h>
#include <string.h>
#include <stdio.h>
#ifdef _OPENMP
#include <omp.h>
#endif
#if defined(__AVX2__) && defined(__FMA__)
#include <immintrin.h>
#endif

enum { QI_F32 = 0, QI_BF16 = 1, QI_I8 = 2 };

typedef struct {
    int fmt;            /* QI_F32 | QI_BF16 | QI_I8 */
    int N, K;           /* rows (outputs) x columns (inputs) */
    const void *w;      /* [N][K] in fmt */
    const float *sc;    /* QI_I8: one scale per row; NULL otherwise */
    int ld;             /* row stride in elements, 0 = K. Lets attention read one head's
                         * columns of K straight out of the [T][heads*hd] projection. */
} QiMat;

#define QI_MR 6
#define QI_NR 16
#define QI_KC 256
#define QI_MC 192

static inline float qi_bf16(uint16_t h){ union { uint32_t u; float f; } v; v.u = (uint32_t)h << 16; return v.f; }

/* Pack W rows [n0, n0+nr) x columns [k0, k0+kc) as f32, k-major: panel[k*QI_NR + j].
 * Rows past nr are zero so the kernel never branches on the edge. The int8 scale
 * is applied here, so the kernel is the same for every format. */
static void qi_pack_w(float *panel, const QiMat *W, int n0, int nr, int k0, int kc){
    for (int j = 0; j < QI_NR; j++) {
        if (j >= nr) { for (int k = 0; k < kc; k++) panel[k*QI_NR + j] = 0.f; continue; }
        int64_t row = (int64_t)(n0 + j) * (W->ld ? W->ld : W->K) + k0;
        if (W->fmt == QI_F32) {
            const float *s = (const float *)W->w + row;
            for (int k = 0; k < kc; k++) panel[k*QI_NR + j] = s[k];
        } else if (W->fmt == QI_BF16) {
            const uint16_t *s = (const uint16_t *)W->w + row;
            for (int k = 0; k < kc; k++) panel[k*QI_NR + j] = qi_bf16(s[k]);
        } else {
            const int8_t *s = (const int8_t *)W->w + row;
            float sc = W->sc[n0 + j];
            for (int k = 0; k < kc; k++) panel[k*QI_NR + j] = (float)s[k] * sc;
        }
    }
}

/* acc[MR][NR] (+)= X[m0.., k0..k0+kc) . panel ; mr <= MR rows valid */
static void qi_kernel(float *Y, int ldy, const float *X, int ldx, const float *panel,
                      int kc, int mr, int nr, int accumulate){
#if defined(__AVX2__) && defined(__FMA__)
    if (mr == QI_MR) {
        __m256 c00=_mm256_setzero_ps(), c01=_mm256_setzero_ps(), c10=_mm256_setzero_ps(), c11=_mm256_setzero_ps();
        __m256 c20=_mm256_setzero_ps(), c21=_mm256_setzero_ps(), c30=_mm256_setzero_ps(), c31=_mm256_setzero_ps();
        __m256 c40=_mm256_setzero_ps(), c41=_mm256_setzero_ps(), c50=_mm256_setzero_ps(), c51=_mm256_setzero_ps();
        const float *x0 = X, *x1 = X + ldx, *x2 = X + 2*ldx, *x3 = X + 3*ldx, *x4 = X + 4*ldx, *x5 = X + 5*ldx;
        for (int k = 0; k < kc; k++) {
            __m256 b0 = _mm256_loadu_ps(panel + k*QI_NR), b1 = _mm256_loadu_ps(panel + k*QI_NR + 8);
            __m256 a;
            a = _mm256_broadcast_ss(x0 + k); c00 = _mm256_fmadd_ps(a, b0, c00); c01 = _mm256_fmadd_ps(a, b1, c01);
            a = _mm256_broadcast_ss(x1 + k); c10 = _mm256_fmadd_ps(a, b0, c10); c11 = _mm256_fmadd_ps(a, b1, c11);
            a = _mm256_broadcast_ss(x2 + k); c20 = _mm256_fmadd_ps(a, b0, c20); c21 = _mm256_fmadd_ps(a, b1, c21);
            a = _mm256_broadcast_ss(x3 + k); c30 = _mm256_fmadd_ps(a, b0, c30); c31 = _mm256_fmadd_ps(a, b1, c31);
            a = _mm256_broadcast_ss(x4 + k); c40 = _mm256_fmadd_ps(a, b0, c40); c41 = _mm256_fmadd_ps(a, b1, c41);
            a = _mm256_broadcast_ss(x5 + k); c50 = _mm256_fmadd_ps(a, b0, c50); c51 = _mm256_fmadd_ps(a, b1, c51);
        }
        float t[QI_MR][QI_NR];
        _mm256_storeu_ps(t[0], c00); _mm256_storeu_ps(t[0]+8, c01);
        _mm256_storeu_ps(t[1], c10); _mm256_storeu_ps(t[1]+8, c11);
        _mm256_storeu_ps(t[2], c20); _mm256_storeu_ps(t[2]+8, c21);
        _mm256_storeu_ps(t[3], c30); _mm256_storeu_ps(t[3]+8, c31);
        _mm256_storeu_ps(t[4], c40); _mm256_storeu_ps(t[4]+8, c41);
        _mm256_storeu_ps(t[5], c50); _mm256_storeu_ps(t[5]+8, c51);
        for (int i = 0; i < QI_MR; i++)
            for (int j = 0; j < nr; j++)
                Y[(int64_t)i*ldy + j] = accumulate ? Y[(int64_t)i*ldy + j] + t[i][j] : t[i][j];
        return;
    }
#endif
    float t[QI_MR][QI_NR];
    memset(t, 0, sizeof t);
    for (int k = 0; k < kc; k++) {
        const float *b = panel + k*QI_NR;
        for (int i = 0; i < mr; i++) {
            float a = X[(int64_t)i*ldx + k];
            for (int j = 0; j < QI_NR; j++) t[i][j] += a * b[j];
        }
    }
    for (int i = 0; i < mr; i++)
        for (int j = 0; j < nr; j++)
            Y[(int64_t)i*ldy + j] = accumulate ? Y[(int64_t)i*ldy + j] + t[i][j] : t[i][j];
}

/* Y[M][N] = X[M][K] . W^T (+ bias). ldx/ldy are the row strides (K and N for
 * dense rows; larger when X or Y is a column slice of a wider matrix). */
static void qi_gemm_ld(float *Y, int ldy, const float *X, int ldx, int M, const QiMat *W, const float *bias){
    const int N = W->N, K = W->K;
    if (M <= 0 || N <= 0) return;
    const int nblocks = (N + QI_NR - 1) / QI_NR;
    const int mblocks = (M + QI_MC - 1) / QI_MC;
    /* Parallel over (m-block, n-panel) tiles; each tile walks all of K, so the
     * sum for one output is always accumulated in the same order: the result
     * does not depend on the thread count. */
    #pragma omp parallel
    {
        /* plain malloc: the kernel loads unaligned, and aligned_alloc is missing from
         * the Windows CRT */
        float *panel = (float *)malloc((size_t)QI_KC * QI_NR * sizeof(float));
        if (!panel) { fprintf(stderr, "OOM qi_gemm panel\n"); exit(1); }
        #pragma omp for schedule(dynamic, 1) collapse(2)
        for (int mb = 0; mb < mblocks; mb++)
            for (int nb = 0; nb < nblocks; nb++) {
                int m0 = mb * QI_MC, mc = M - m0 < QI_MC ? M - m0 : QI_MC;
                int n0 = nb * QI_NR, nr = N - n0 < QI_NR ? N - n0 : QI_NR;
                for (int k0 = 0; k0 < K; k0 += QI_KC) {
                    int kc = K - k0 < QI_KC ? K - k0 : QI_KC;
                    qi_pack_w(panel, W, n0, nr, k0, kc);
                    for (int i = 0; i < mc; i += QI_MR) {
                        int mr = mc - i < QI_MR ? mc - i : QI_MR;
                        qi_kernel(Y + (int64_t)(m0 + i) * ldy + n0, ldy,
                                  X + (int64_t)(m0 + i) * ldx + k0, ldx,
                                  panel, kc, mr, nr, k0 > 0);
                    }
                }
                if (bias)
                    for (int i = 0; i < mc; i++)
                        for (int j = 0; j < nr; j++) Y[(int64_t)(m0 + i) * ldy + n0 + j] += bias[n0 + j];
            }
        free(panel);
    }
}

static inline void qi_gemm(float *Y, const float *X, int M, const QiMat *W, const float *bias){
    qi_gemm_ld(Y, W->N, X, W->K, M, W, bias);
}

/* ---- int8 x int8 (opt-in): the activations quantized per row too ---------
 *
 * With VNNI one vpdpbusd does four multiply-adds per 32-bit lane where an FMA
 * does one, so a DiT step is compute-bound on a quarter of the instructions.
 * The price is an 8-bit activation (one scale per row, amax/127): that is a
 * quality question, measured against the oracle, not assumed. vpdpbusd wants
 * unsigned x signed bytes, so x is stored as q+128 and 128*sum(w) comes off the
 * int32 sum at the end. The int32 sum over the whole K cannot overflow:
 * K * 255 * 127 stays under 2^31 up to K = 66000.
 *
 * Only for QI_I8 weights and only where VNNI exists (AVX-VNNI, or AVX-512 VNNI
 * with VL); qi_gemm_act8() falls back to qi_gemm() otherwise. */
#if (defined(__AVXVNNI__) || (defined(__AVX512VNNI__) && defined(__AVX512VL__))) && defined(__AVX2__)
#define QI_HAVE_VNNI 1
#if defined(__AVX512VNNI__) && defined(__AVX512VL__)
#define QI_DPBUSD(acc, a, b) _mm256_dpbusd_epi32(acc, a, b)
#else
#define QI_DPBUSD(acc, a, b) _mm256_dpbusd_avx_epi32(acc, a, b)
#endif

/* W rows [n0, n0+nr), all K (padded to Kp, a multiple of 4), as [Kp/4][16][4]
 * bytes, plus the sum of each row for the +128 correction. */
static void qi_pack_w8(int8_t *panel, int32_t *wsum, const QiMat *W, int n0, int nr, int Kp){
    const int K = W->K, ld = W->ld ? W->ld : K;
    for (int j = 0; j < QI_NR; j++) {
        int32_t s = 0;
        const int8_t *r = j < nr ? (const int8_t *)W->w + (int64_t)(n0 + j) * ld : NULL;
        for (int k = 0; k < Kp; k++) {
            int8_t v = r && k < K ? r[k] : 0;
            panel[(k >> 2) * (QI_NR * 4) + j * 4 + (k & 3)] = v;
            s += v;
        }
        wsum[j] = s;
    }
}

static void qi_kernel8(float *Y, int ldy, const uint8_t *X, int Kp, const int8_t *panel,
                       const int32_t *wsum, const float *sx, const float *sw, int mr, int nr, const float *bias){
    __m256i c[QI_MR][2];
    for (int i = 0; i < QI_MR; i++) c[i][0] = c[i][1] = _mm256_setzero_si256();
    const int kq = Kp >> 2;
    if (mr == QI_MR) {
        for (int q = 0; q < kq; q++) {
            __m256i b0 = _mm256_loadu_si256((const __m256i *)(panel + q * 64));
            __m256i b1 = _mm256_loadu_si256((const __m256i *)(panel + q * 64 + 32));
            for (int i = 0; i < QI_MR; i++) {
                int32_t a4; memcpy(&a4, X + (int64_t)i * Kp + q * 4, 4);
                __m256i a = _mm256_set1_epi32(a4);
                c[i][0] = QI_DPBUSD(c[i][0], a, b0);
                c[i][1] = QI_DPBUSD(c[i][1], a, b1);
            }
        }
    } else {
        for (int q = 0; q < kq; q++) {
            __m256i b0 = _mm256_loadu_si256((const __m256i *)(panel + q * 64));
            __m256i b1 = _mm256_loadu_si256((const __m256i *)(panel + q * 64 + 32));
            for (int i = 0; i < mr; i++) {
                int32_t a4; memcpy(&a4, X + (int64_t)i * Kp + q * 4, 4);
                __m256i a = _mm256_set1_epi32(a4);
                c[i][0] = QI_DPBUSD(c[i][0], a, b0);
                c[i][1] = QI_DPBUSD(c[i][1], a, b1);
            }
        }
    }
    int32_t t[QI_NR];
    for (int i = 0; i < mr; i++) {
        _mm256_storeu_si256((__m256i *)t, c[i][0]);
        _mm256_storeu_si256((__m256i *)(t + 8), c[i][1]);
        for (int j = 0; j < nr; j++)
            Y[(int64_t)i * ldy + j] = (float)(t[j] - 128 * wsum[j]) * sx[i] * sw[j] + (bias ? bias[j] : 0.f);
    }
}
#endif

/* Y = X . W^T with X quantized to int8 per row. W must be QI_I8. */
static void qi_gemm_act8(float *Y, int ldy, const float *X, int ldx, int M, const QiMat *W, const float *bias){
#ifdef QI_HAVE_VNNI
    const int N = W->N, K = W->K, Kp = (K + 3) & ~3;
    if (W->fmt != QI_I8 || M <= 0 || N <= 0) { qi_gemm_ld(Y, ldy, X, ldx, M, W, bias); return; }
    uint8_t *xq = (uint8_t *)malloc((size_t)M * Kp);
    float *sx = (float *)malloc(sizeof(float) * (size_t)M);
    if (!xq || !sx) { fprintf(stderr, "OOM qi_gemm_act8\n"); exit(1); }
    #pragma omp parallel for schedule(static)
    for (int m = 0; m < M; m++) {
        const float *r = X + (int64_t)m * ldx;
        float am = 0.f;
        for (int k = 0; k < K; k++) { float a = r[k] < 0 ? -r[k] : r[k]; if (a > am) am = a; }
        float s = am > 1e-30f ? am / 127.f : 1.f, inv = 1.f / s;
        sx[m] = s;
        uint8_t *d = xq + (int64_t)m * Kp;
        for (int k = 0; k < K; k++) { float v = r[k] * inv; int iv = (int)(v < 0 ? v - 0.5f : v + 0.5f); d[k] = (uint8_t)(iv + 128); }
        for (int k = K; k < Kp; k++) d[k] = 128;
    }
    const int nblocks = (N + QI_NR - 1) / QI_NR, mblocks = (M + QI_MC - 1) / QI_MC;
    #pragma omp parallel
    {
        int8_t *panel = (int8_t *)malloc((size_t)Kp * QI_NR);
        int32_t wsum[QI_NR];
        if (!panel) { fprintf(stderr, "OOM qi_gemm_act8 panel\n"); exit(1); }
        #pragma omp for schedule(dynamic, 1) collapse(2)
        for (int nb = 0; nb < nblocks; nb++)
            for (int mb = 0; mb < mblocks; mb++) {
                int n0 = nb * QI_NR, nr = N - n0 < QI_NR ? N - n0 : QI_NR;
                int m0 = mb * QI_MC, mc = M - m0 < QI_MC ? M - m0 : QI_MC;
                qi_pack_w8(panel, wsum, W, n0, nr, Kp);
                for (int i = 0; i < mc; i += QI_MR) {
                    int mr = mc - i < QI_MR ? mc - i : QI_MR;
                    qi_kernel8(Y + (int64_t)(m0 + i) * ldy + n0, ldy, xq + (int64_t)(m0 + i) * Kp, Kp,
                               panel, wsum, sx + m0 + i, W->sc + n0, mr, nr, bias ? bias + n0 : NULL);
                }
            }
        free(panel);
    }
    free(xq); free(sx);
#else
    qi_gemm_ld(Y, ldy, X, ldx, M, W, bias);
#endif
}

/* qi_quantize_i8 of a bf16 matrix, row by row: bf16 to f32 is exact, so the rows and
 * scales are qi_quantize_i8's of the f32 copy, without the copy (the loader's time and
 * its biggest transient buffer). */
static void qi_quantize_i8_bf16(const uint16_t *src, int N, int K, int8_t *q, float *sc){
    #pragma omp parallel for schedule(static)
    for (int n = 0; n < N; n++) {
        const uint16_t *r = src + (int64_t)n * K;
        float am = 0.f;
        for (int k = 0; k < K; k++) { float x = qi_bf16(r[k]), a = x < 0 ? -x : x; if (a > am) am = a; }
        float s = am > 1e-12f ? am / 127.f : 1.f, inv = 1.f / s;
        sc[n] = s;
        int8_t *d = q + (int64_t)n * K;
        for (int k = 0; k < K; k++) {
            float v = qi_bf16(r[k]) * inv;
            int iv = (int)(v < 0 ? v - 0.5f : v + 0.5f);
            if (iv > 127) iv = 127;
            if (iv < -127) iv = -127;
            d[k] = (int8_t)iv;
        }
    }
}

/* Per-row int8 quantization of an f32 matrix (max-abs / 127, round to nearest). */
static void qi_quantize_i8(const float *src, int N, int K, int8_t *q, float *sc){
    #pragma omp parallel for schedule(static)
    for (int n = 0; n < N; n++) {
        const float *r = src + (int64_t)n * K;
        float am = 0.f;
        for (int k = 0; k < K; k++) { float a = r[k] < 0 ? -r[k] : r[k]; if (a > am) am = a; }
        float s = am > 1e-12f ? am / 127.f : 1.f, inv = 1.f / s;
        sc[n] = s;
        int8_t *d = q + (int64_t)n * K;
        for (int k = 0; k < K; k++) {
            float v = r[k] * inv;
            int iv = (int)(v < 0 ? v - 0.5f : v + 0.5f);
            if (iv > 127) iv = 127;
            if (iv < -127) iv = -127;
            d[k] = (int8_t)iv;
        }
    }
}

#endif