Skip to content

Run frontier MoE models on hardware you already own — pure C, zero deps, experts streamed from disk. Tiny engine, immense model. 🐦

C
22.6 KB
Raw Blame
/* expert_ffn.h — the routed-expert FFN kernel shared by the MoE engines.
 *
 * One weight layout, one activation contract, one way of running a layer:
 *
 *   layout   "planar unsigned int4, gs=64": a row of I weights (I % 64 == 0) is
 *            I/64 blocks of 32 bytes; byte k of a block holds element k in its
 *            low nibble and element k+32 in its high nibble, both stored as
 *            v+8 (0..15). One f32 scale per (row, block), row-major [O][I/64].
 *            The same layout quant.h calls "fmt 4 planar" (K1b): `and 0x0F`
 *            yields elements 0..31 already in order, `srli 4` the other 32,
 *            so no unpack instruction is ever paid, and the unsigned nibbles
 *            feed vpdpbusd natively through dot(v,x) = dot(u,x) - 8*sum(x).
 *   f32      activations stay f32 (default): the weight is exact in f32, one
 *            fma per element, one fma per block for the scale. The only
 *            difference from a pair-layout f32 kernel is accumulation order.
 *   i8       activations quantized once per token to int8 with one scale per
 *            row plus the per-block sums (opt-in, `mode` 1): integer dots,
 *            2 dpbusd per block. Not bit-identical to f32: same policy as
 *            IDOT elsewhere in the tree (opt-in until an ablation blesses it).
 *   layer    xf_moe_run() runs a whole MoE layer: gate+up fused over the
 *            experts the routing touched, silu, down, then the K contributions
 *            of each token summed in rank order. Threads split (expert, row
 *            chunk) items, so a layer costs two OpenMP regions instead of
 *            3*K, and a prompt row never re-reads a weight another row of the
 *            same expert just used.
 *
 * Why it exists: measured on the real Qwen3.6 container at full residency the
 * MoE cost 34 ms/token in steady state against a DRAM floor of 17 ms for the
 * int8-unpacked experts it was reading and 9 ms for the int4 bytes actually
 * on disk. Unpacking int4 to int8 at load doubled both the bytes per token
 * and the resident set; per-GEMV OpenMP regions (24 per layer) and one
 * reduction per 64-block did the rest. The same shape (int4 experts, f32
 * hidden state, top-k routing) is every engine here, which is why this is a
 * header and not a function in one of them.
 *
 * Header-only, all static, pure compute: no Model, no I/O, no allocation
 * beyond what the caller hands in. AVX2/FMA, AVX-VNNI or AVX512-VNNI where built,
 * with a scalar fallback that defines the numerics every SIMD path must
 * reproduce (f32 paths: same per-lane accumulation order; i8 paths: exact
 * integer sums, so identical by construction). */
#ifndef COLI_EXPERT_FFN_H
#define COLI_EXPERT_FFN_H

#include <stdint.h>
#include <stddef.h>
#include <string.h>
#include <math.h>

#if defined(__x86_64__) || defined(_M_X64) || defined(__i386__)
#include <immintrin.h>
#endif
#ifdef _OPENMP
#include <omp.h>
#endif

#define XF_BLOCK 64                       /* elements per planar block and per scale group */
#define XF_BLOCK_BYTES 32

/* ---- layout helpers ------------------------------------------------------- */

static inline size_t xf_row_bytes(int I) { return (size_t)I / 2; }
static inline int    xf_groups(int I)    { return I / XF_BLOCK; }
static inline int    xf_layout_ok(int I) { return I > 0 && (I % XF_BLOCK) == 0; }

/* Repack one row from the "signed pairs" container layout (byte j = element
 * 2j in the low nibble, 2j+1 in the high nibble, two's-complement 4-bit — the
 * qwen36 converter's pack_int4) into planar unsigned. src and dst may not
 * overlap. I % 64 == 0. */
static inline void xf_repack_row_pairs_signed(uint8_t *dst, const uint8_t *src, int I) {
    int b = 0;
#if defined(__AVX2__)
    /* 16 pair bytes hold elements 0..31: the low nibbles are the even ones,
     * the high nibbles the odd ones, so interleaving the two nibble vectors
     * byte-wise puts elements 0..31 in order. Same for 32..63 from the next 16
     * bytes; xor 8 turns the signed nibble into v+8; the block is lo | hi<<4. */
    const __m128i m4 = _mm_set1_epi8(0x0F);
    const __m256i e8 = _mm256_set1_epi8(8);
    for (; b < I / XF_BLOCK; b++) {
        const uint8_t *s = src + (size_t)b * XF_BLOCK_BYTES;
        __m128i a = _mm_loadu_si128((const __m128i *)s), c = _mm_loadu_si128((const __m128i *)(s + 16));
        __m128i al = _mm_and_si128(a, m4), ah = _mm_and_si128(_mm_srli_epi16(a, 4), m4);
        __m128i cl = _mm_and_si128(c, m4), ch = _mm_and_si128(_mm_srli_epi16(c, 4), m4);
        __m256i e0 = _mm256_set_m128i(_mm_unpackhi_epi8(al, ah), _mm_unpacklo_epi8(al, ah));   /* elements 0..31 */
        __m256i e1 = _mm256_set_m128i(_mm_unpackhi_epi8(cl, ch), _mm_unpacklo_epi8(cl, ch));   /* elements 32..63 */
        e0 = _mm256_xor_si256(e0, e8); e1 = _mm256_xor_si256(e1, e8);
        _mm256_storeu_si256((__m256i *)(dst + (size_t)b * XF_BLOCK_BYTES), _mm256_or_si256(e0, _mm256_slli_epi16(e1, 4)));
    }
#endif
    for (; b < I / XF_BLOCK; b++) {
        const uint8_t *s = src + (size_t)b * XF_BLOCK_BYTES;
        uint8_t *d = dst + (size_t)b * XF_BLOCK_BYTES;
        for (int k = 0; k < 32; k++) {
            /* element k of the block: byte k/2, nibble k&1; element k+32: byte 16 + k/2 */
            unsigned lo = (unsigned)(s[k >> 1]      >> ((k & 1) * 4)) & 0xF;
            unsigned hi = (unsigned)(s[16 + (k >> 1)] >> ((k & 1) * 4)) & 0xF;
            /* signed 4-bit -> v+8 is the nibble xor 8 */
            d[k] = (uint8_t)((lo ^ 8u) | ((hi ^ 8u) << 4));
        }
    }
}
static inline void xf_repack_pairs_signed(uint8_t *dst, const uint8_t *src, int O, int I) {
    for (int o = 0; o < O; o++)
        xf_repack_row_pairs_signed(dst + (size_t)o * xf_row_bytes(I), src + (size_t)o * xf_row_bytes(I), I);
}
/* Decode one element from a planar row (tests and scalar paths). */
static inline int xf_get(const uint8_t *row, int i) {
    const uint8_t *blk = row + (size_t)(i / XF_BLOCK) * XF_BLOCK_BYTES;
    int k = i % XF_BLOCK;
    unsigned u = k < 32 ? (blk[k] & 0xF) : (blk[k - 32] >> 4);
    return (int)u - 8;
}

/* ---- activation contract for the i8 mode ---------------------------------- */

/* Quantize x[I] to int8 with one scale (amax/127), and the per-block sums as
 * f32 (exact: |sum| <= 64*127). Returns the scale. */
static inline float xf_act_i8(const float *x, int I, int8_t *xq, float *xsum) {
    float am = 0.f;
    for (int i = 0; i < I; i++) { float a = fabsf(x[i]); if (a > am) am = a; }
    float s = am > 0.f ? am / 127.f : 1.f, inv = 1.f / s;
    for (int i = 0; i < I; i++) xq[i] = (int8_t)lrintf(x[i] * inv);
    for (int g = 0; g < I / XF_BLOCK; g++) {
        int32_t t = 0;
        for (int i = 0; i < XF_BLOCK; i++) t += xq[g * XF_BLOCK + i];
        xsum[g] = (float)t;
    }
    return s;
}

/* ---- scalar reference kernels (the numerics contract) --------------------- */

/* f32: per block, 8 lanes l=0..7 accumulate elements l, l+8, ..., l+56 in
 * order with fma; the 8 lane sums are scaled by the block scale into 8 lane
 * accumulators; the row ends with the fixed 8->1 tree of xf_hsum8. The SIMD
 * paths reproduce exactly this. */
static inline float xf_hsum8_scalar(const float *l) {
    float a0 = l[0] + l[4], a1 = l[1] + l[5], a2 = l[2] + l[6], a3 = l[3] + l[7];
    float b0 = a0 + a2, b1 = a1 + a3;
    return b0 + b1;
}
static inline float xf_dot_f32_ref(const uint8_t *w, const float *sc, const float *x, int I) {
    float acc[8] = {0, 0, 0, 0, 0, 0, 0, 0};
    for (int g = 0; g < I / XF_BLOCK; g++) {
        const uint8_t *blk = w + (size_t)g * XF_BLOCK_BYTES;
        const float *xb = x + g * XF_BLOCK;
        float lane[8] = {0, 0, 0, 0, 0, 0, 0, 0};
        for (int i = 0; i < XF_BLOCK; i++) {
            int v = (i < 32 ? (blk[i] & 0xF) : (blk[i - 32] >> 4)) - 8;
            lane[i & 7] = fmaf((float)v, xb[i], lane[i & 7]);
        }
        for (int l = 0; l < 8; l++) acc[l] = fmaf(lane[l], sc[g], acc[l]);
    }
    return xf_hsum8_scalar(acc);
}
/* i8: y = sx * sum_g sc[g] * (dot_u(g) - 8*xsum[g]); the per-block integer
 * dot is exact, then one f32 fma per block per lane exactly as the f32 path
 * (the 8 lanes hold the 8 int32 partial sums vpdpbusd produces: lane l sums
 * elements 4l..4l+3 and 32+4l..32+4l+3 of the block). */
static inline float xf_dot_i8_ref(const uint8_t *w, const float *sc, const int8_t *xq, const float *xsum, float sx, int I) {
    float acc[8] = {0, 0, 0, 0, 0, 0, 0, 0};
    float corr = 0.f;
    for (int g = 0; g < I / XF_BLOCK; g++) {
        const uint8_t *blk = w + (size_t)g * XF_BLOCK_BYTES;
        const int8_t *xb = xq + g * XF_BLOCK;
        int32_t lane[8] = {0, 0, 0, 0, 0, 0, 0, 0};
        for (int k = 0; k < 32; k++) {
            lane[k >> 2] += (int32_t)(blk[k] & 0xF) * xb[k];
            lane[k >> 2] += (int32_t)(blk[k] >> 4)  * xb[k + 32];
        }
        for (int l = 0; l < 8; l++) acc[l] = fmaf((float)lane[l], sc[g], acc[l]);
        corr = fmaf(sc[g], xsum[g], corr);
    }
    return (xf_hsum8_scalar(acc) - 8.f * corr) * sx;
}

/* ---- SIMD kernels ---------------------------------------------------------- */

#if defined(__AVX2__) && defined(__FMA__)
#define XF_HAVE_AVX2 1
static inline float xf_hsum8(__m256 v) {
    __m128 lo = _mm256_castps256_ps128(v), hi = _mm256_extractf128_ps(v, 1);
    __m128 a = _mm_add_ps(lo, hi);                       /* l+l+4 */
    __m128 b = _mm_add_ps(a, _mm_movehl_ps(a, a));       /* (0+4)+(2+6), (1+5)+(3+7) */
    __m128 c = _mm_add_ss(b, _mm_shuffle_ps(b, b, 1));
    return _mm_cvtss_f32(c);
}
/* nibbles of one 32-byte block as two 32-lane signed int8 vectors (v = u-8) */
static inline void xf_nib_s8(__m256i bb, __m256i *lo, __m256i *hi) {
    const __m256i m4 = _mm256_set1_epi8(0x0F), e8 = _mm256_set1_epi8(8);
    *lo = _mm256_sub_epi8(_mm256_and_si256(bb, m4), e8);
    *hi = _mm256_sub_epi8(_mm256_and_si256(_mm256_srli_epi16(bb, 4), m4), e8);
}
/* No 512-bit f32 path on purpose: element i must land in lane i&7 in the
 * reference's order, and folding 16 lanes onto 8 changes the fma order. The
 * i8 path is exact integers and could widen; it is DRAM-bound already. */

/* f32 row dot, AVX2: 8 lanes per element octet, lane = i&7 */
static inline float xf_dot_f32_avx2(const uint8_t *w, const float *sc, const float *x, int I) {
    __m256 acc = _mm256_setzero_ps();
    for (int g = 0; g < I / XF_BLOCK; g++) {
        __m256i bb = _mm256_loadu_si256((const __m256i *)(w + (size_t)g * XF_BLOCK_BYTES));
        __m256i lo, hi; xf_nib_s8(bb, &lo, &hi);
        const float *xb = x + g * XF_BLOCK;
        __m128i lo0 = _mm256_castsi256_si128(lo), lo1 = _mm256_extracti128_si256(lo, 1);
        __m128i hi0 = _mm256_castsi256_si128(hi), hi1 = _mm256_extracti128_si256(hi, 1);
        __m256 lane = _mm256_setzero_ps();
        lane = _mm256_fmadd_ps(_mm256_cvtepi32_ps(_mm256_cvtepi8_epi32(lo0)),                    _mm256_loadu_ps(xb),      lane);
        lane = _mm256_fmadd_ps(_mm256_cvtepi32_ps(_mm256_cvtepi8_epi32(_mm_srli_si128(lo0, 8))), _mm256_loadu_ps(xb + 8),  lane);
        lane = _mm256_fmadd_ps(_mm256_cvtepi32_ps(_mm256_cvtepi8_epi32(lo1)),                    _mm256_loadu_ps(xb + 16), lane);
        lane = _mm256_fmadd_ps(_mm256_cvtepi32_ps(_mm256_cvtepi8_epi32(_mm_srli_si128(lo1, 8))), _mm256_loadu_ps(xb + 24), lane);
        lane = _mm256_fmadd_ps(_mm256_cvtepi32_ps(_mm256_cvtepi8_epi32(hi0)),                    _mm256_loadu_ps(xb + 32), lane);
        lane = _mm256_fmadd_ps(_mm256_cvtepi32_ps(_mm256_cvtepi8_epi32(_mm_srli_si128(hi0, 8))), _mm256_loadu_ps(xb + 40), lane);
        lane = _mm256_fmadd_ps(_mm256_cvtepi32_ps(_mm256_cvtepi8_epi32(hi1)),                    _mm256_loadu_ps(xb + 48), lane);
        lane = _mm256_fmadd_ps(_mm256_cvtepi32_ps(_mm256_cvtepi8_epi32(_mm_srli_si128(hi1, 8))), _mm256_loadu_ps(xb + 56), lane);
        acc = _mm256_fmadd_ps(lane, _mm256_set1_ps(sc[g]), acc);
    }
    return xf_hsum8(acc);
}

/* i8 row dot: unsigned nibbles x int8 activations, 8 int32 lanes per block */
#if defined(__AVXVNNI__) || (defined(__AVX512VNNI__) && defined(__AVX512VL__))
#define XF_HAVE_VNNI 1
#if defined(__AVXVNNI__) && !defined(__AVX512VNNI__)
#define xf_dpbusd(acc, u, x) _mm256_dpbusd_avx_epi32(acc, u, x)
#else
#define xf_dpbusd(acc, u, x) _mm256_dpbusd_epi32(acc, u, x)
#endif
#else
static inline __m256i xf_dpbusd(__m256i acc, __m256i u, __m256i x) {
    /* u <= 15, |x| <= 127: a pair sum fits int16 (<= 3810) */
    return _mm256_add_epi32(acc, _mm256_madd_epi16(_mm256_maddubs_epi16(u, x), _mm256_set1_epi16(1)));
}
#endif
static inline float xf_dot_i8_avx2(const uint8_t *w, const float *sc, const int8_t *xq, const float *xsum, float sx, int I) {
    const __m256i m4 = _mm256_set1_epi8(0x0F), z = _mm256_setzero_si256();
    __m256 acc = _mm256_setzero_ps();
    int ng = I / XF_BLOCK, g = 0;
    for (; g < ng; g++) {
        __m256i bb = _mm256_loadu_si256((const __m256i *)(w + (size_t)g * XF_BLOCK_BYTES));
        const int8_t *xb = xq + g * XF_BLOCK;
        __m256i d = xf_dpbusd(z, _mm256_and_si256(bb, m4), _mm256_loadu_si256((const __m256i *)xb));
        d = xf_dpbusd(d, _mm256_and_si256(_mm256_srli_epi16(bb, 4), m4), _mm256_loadu_si256((const __m256i *)(xb + 32)));
        acc = _mm256_fmadd_ps(_mm256_cvtepi32_ps(d), _mm256_set1_ps(sc[g]), acc);
    }
    /* corr = sum_g sc[g]*xsum[g] in the scalar reference's order (sequential fma) */
    float c = 0.f;
    for (g = 0; g < ng; g++) c = fmaf(sc[g], xsum[g], c);
    return (xf_hsum8(acc) - 8.f * c) * sx;
}
#endif /* AVX2 */

/* ---- dispatch -------------------------------------------------------------- */

static inline float xf_dot_f32(const uint8_t *w, const float *sc, const float *x, int I) {
#ifdef XF_HAVE_AVX2
    return xf_dot_f32_avx2(w, sc, x, I);
#else
    return xf_dot_f32_ref(w, sc, x, I);
#endif
}
static inline float xf_dot_i8(const uint8_t *w, const float *sc, const int8_t *xq, const float *xsum, float sx, int I) {
#ifdef XF_HAVE_AVX2
    return xf_dot_i8_avx2(w, sc, xq, xsum, sx, I);
#else
    return xf_dot_i8_ref(w, sc, xq, xsum, sx, I);
#endif
}

/* ---- one expert ------------------------------------------------------------- */

typedef struct {
    const uint8_t *g4, *u4, *d4;   /* planar rows: gate/up [F][H], down [H][F] */
    const float *gs, *us, *ds;     /* block scales: [F][H/64], [F][H/64], [H][F/64] */
} XfExpert;

/* Activations of one token, prepared once per layer (both modes carry x). */
typedef struct {
    const float *x;      /* [H] f32 */
    const int8_t *xq;    /* [H] int8, i8 mode */
    const float *xsum;   /* [H/64] block sums, i8 mode */
    float sx;            /* activation scale, i8 mode */
} XfAct;

/* gate+up rows [r0,r1) of one expert for one token: g[r], u[r]. Both matrices
 * share the activation, so their rows are interleaved in the same pass. */
static inline void xf_gate_up_rows(float *g, float *u, const XfExpert *e, const XfAct *a, int H, int r0, int r1, int mode) {
    size_t rb = xf_row_bytes(H); int ng = xf_groups(H);
    for (int r = r0; r < r1; r++) {
        const uint8_t *wg = e->g4 + (size_t)r * rb, *wu = e->u4 + (size_t)r * rb;
        const float *sg = e->gs + (size_t)r * ng, *su = e->us + (size_t)r * ng;
        if (mode) { g[r] = xf_dot_i8(wg, sg, a->xq, a->xsum, a->sx, H); u[r] = xf_dot_i8(wu, su, a->xq, a->xsum, a->sx, H); }
        else      { g[r] = xf_dot_f32(wg, sg, a->x, H);                 u[r] = xf_dot_f32(wu, su, a->x, H); }
    }
}
/* silu(g)*u over [0,F): the hidden row, in the engines' historical form */
static inline void xf_swiglu(float *h, const float *g, const float *u, int F) {
    for (int i = 0; i < F; i++) { float gv = g[i]; h[i] = (gv / (1.f + expf(-gv))) * u[i]; }
}
/* down rows [r0,r1) of one expert for one hidden row */
static inline void xf_down_rows(float *y, const XfExpert *e, const XfAct *h, int F, int r0, int r1, int mode) {
    size_t rb = xf_row_bytes(F); int ng = xf_groups(F);
    for (int r = r0; r < r1; r++) {
        const uint8_t *wd = e->d4 + (size_t)r * rb; const float *sd = e->ds + (size_t)r * ng;
        y[r] = mode ? xf_dot_i8(wd, sd, h->xq, h->xsum, h->sx, F) : xf_dot_f32(wd, sd, h->x, F);
    }
}

/* ---- one layer ---------------------------------------------------------------
 *
 * S tokens, K routed experts each. idx[s*K+k] is the expert id (or -1), val the
 * router weight, experts[s*K+k] the resident expert (NULL iff idx < 0).
 * out[s][H] = sum_k val[s][k] * expert_k(x[s]) accumulated in k order, as the
 * per-token loops it replaces did. xf_moe_add adds that sum into out instead of
 * overwriting it: a caller that cuts one layer into several runs (qwen36, when
 * its cache holds fewer experts than the batch routes to) sends every cut
 * through the same statement, so the bits cannot depend on where it cut, whether
 * or not the compiler fuses that multiply-add. xf_moe_run zeroes out first.
 * Callers own `scratch` (xf_moe_scratch_bytes).
 *
 * Work is split into (expert, row chunk) items twice: gate+up over F rows, then
 * down over H rows. A prompt of S rows routed to the same expert reads that
 * expert's weights from DRAM once per chunk, the other rows hit cache. */
static inline size_t xf_moe_scratch_bytes(int S, int K, int H, int F) {
    size_t n = (size_t)S * K;
    return n * (2 * (size_t)F + (size_t)H) * sizeof(float)      /* g,u per (s,k); contrib per (s,k) */
         + n * (size_t)F * sizeof(float)                           /* h per (s,k) */
         + n * ((size_t)F + 64) + n * (size_t)(F / XF_BLOCK) * sizeof(float) /* hq, hsum (i8 mode) */
         + (size_t)S * ((size_t)H + 64) + (size_t)S * (size_t)(H / XF_BLOCK) * sizeof(float) /* xq, xsum */
         + n * sizeof(int) * 4 + 4096;
}

/* The per-(s,k) expert outputs alone: ctb[(s*K+k)*H ..] = expert_k(x[s]) for every
 * pair with an expert, nothing summed. ctb may be the caller's (an engine that adds
 * other outputs in the same rank order, vk_tier.h) or NULL: then it is carved from
 * scratch. Returns it, or NULL when no pair has an expert (nothing written). */
static inline float *xf_moe_compute(float *ctb_out, const float *x, int S, int K, int H, int F,
                                    const int *idx, const XfExpert *const *experts,
                                    int mode, void *scratch) {
    const size_t n = (size_t)S * K;
    char *p = (char *)scratch;
#define XF_TAKE(T, count) ((T *)p); p += (((size_t)(count) * sizeof(T)) + 63) & ~(size_t)63
    float *g    = XF_TAKE(float, n * F);
    float *u    = XF_TAKE(float, n * F);
    float *h    = XF_TAKE(float, n * F);
    float *ctb  = XF_TAKE(float, n * H);
    int8_t *hq  = XF_TAKE(int8_t, n * F);
    float *hsum = XF_TAKE(float, n * (F / XF_BLOCK));
    float *hsx  = XF_TAKE(float, n);
    int8_t *xq  = XF_TAKE(int8_t, (size_t)S * H);
    float *xsum = XF_TAKE(float, (size_t)S * (H / XF_BLOCK));
    float *xsx  = XF_TAKE(float, S);
    int *uniq   = XF_TAKE(int, n);          /* distinct expert ids, in first-seen order */
    int *head   = XF_TAKE(int, n);          /* first (s,k) pair index of each distinct expert */
    int *next   = XF_TAKE(int, n);          /* chain of pairs sharing an expert */
    int *cnt    = XF_TAKE(int, n);
#undef XF_TAKE
    if (ctb_out) ctb = ctb_out;

    /* activations once per token */
    if (mode) for (int s = 0; s < S; s++) xsx[s] = xf_act_i8(x + (size_t)s * H, H, xq + (size_t)s * H, xsum + (size_t)s * (H / XF_BLOCK));

    /* group the (s,k) pairs by expert */
    int nu = 0;
    for (int s = 0; s < S; s++) for (int k = 0; k < K; k++) {
        int i = s * K + k; next[i] = -1;
        if (idx[i] < 0 || !experts[i]) continue;
        int j = 0; for (; j < nu; j++) if (uniq[j] == idx[i]) break;
        if (j == nu) { uniq[nu] = idx[i]; head[nu] = i; cnt[nu] = 1; nu++; }
        else { int t = head[j]; while (next[t] >= 0) t = next[t]; next[t] = i; cnt[j]++; }
    }
    if (nu == 0) return NULL;

    int T = 1;
#ifdef _OPENMP
    T = omp_get_max_threads();
#endif
    /* chunking: enough items to feed every thread, rows in multiples of 8 */
    int cF = (4 * T + nu - 1) / nu; if (cF < 1) cF = 1; if (cF > F / 8) cF = F / 8 > 0 ? F / 8 : 1;
    int cH = (4 * T + nu - 1) / nu; if (cH < 1) cH = 1; if (cH > H / 8) cH = H / 8 > 0 ? H / 8 : 1;
    int rowsF = (F + cF - 1) / cF; rowsF = (rowsF + 7) & ~7; cF = (F + rowsF - 1) / rowsF;
    int rowsH = (H + cH - 1) / cH; rowsH = (rowsH + 7) & ~7; cH = (H + rowsH - 1) / rowsH;

    /* phase 1: gate+up for every (expert, chunk), all rows routed to that expert */
    #pragma omp parallel for schedule(dynamic, 1)
    for (int it = 0; it < nu * cF; it++) {
        int e = it / cF, c = it % cF;
        int r0 = c * rowsF, r1 = r0 + rowsF; if (r1 > F) r1 = F;
        for (int i = head[e]; i >= 0; i = next[i]) {
            int s = i / K;
            XfAct a = { x + (size_t)s * H, xq + (size_t)s * H, xsum + (size_t)s * (H / XF_BLOCK), mode ? xsx[s] : 0.f };
            xf_gate_up_rows(g + (size_t)i * F, u + (size_t)i * F, experts[i], &a, H, r0, r1, mode);
        }
    }
    /* hidden rows (cheap, per pair) */
    #pragma omp parallel for schedule(static)
    for (int i = 0; i < (int)n; i++) {
        if (idx[i] < 0 || !experts[i]) continue;
        xf_swiglu(h + (size_t)i * F, g + (size_t)i * F, u + (size_t)i * F, F);
        if (mode) hsx[i] = xf_act_i8(h + (size_t)i * F, F, hq + (size_t)i * F, hsum + (size_t)i * (F / XF_BLOCK));
    }
    /* phase 2: down for every (expert, chunk) */
    #pragma omp parallel for schedule(dynamic, 1)
    for (int it = 0; it < nu * cH; it++) {
        int e = it / cH, c = it % cH;
        int r0 = c * rowsH, r1 = r0 + rowsH; if (r1 > H) r1 = H;
        for (int i = head[e]; i >= 0; i = next[i]) {
            XfAct a = { h + (size_t)i * F, hq + (size_t)i * F, hsum + (size_t)i * (F / XF_BLOCK), mode ? hsx[i] : 0.f };
            xf_down_rows(ctb + (size_t)i * H, experts[i], &a, F, r0, r1, mode);
        }
    }
    return ctb;
}

static inline void xf_moe_add(float *out, const float *x, int S, int K, int H, int F,
                              const int *idx, const float *val, const XfExpert *const *experts,
                              int mode, void *scratch) {
    float *ctb = xf_moe_compute(NULL, x, S, K, H, F, idx, experts, mode, scratch);
    if (!ctb) return;
    /* rank-order sum per token: out[s] += sum_k val[s][k] * ctb[s][k] */
    #pragma omp parallel for schedule(static)
    for (int s = 0; s < S; s++) {
        float *os = out + (size_t)s * H;
        for (int k = 0; k < K; k++) {
            int i = s * K + k; if (idx[i] < 0 || !experts[i]) continue;
            float w = val[i]; const float *c = ctb + (size_t)i * H;
            for (int d = 0; d < H; d++) os[d] += w * c[d];
        }
    }
}

static inline void xf_moe_run(float *out, const float *x, int S, int K, int H, int F,
                              const int *idx, const float *val, const XfExpert *const *experts,
                              int mode, void *scratch) {
    memset(out, 0, (size_t)S * H * sizeof(float));
    xf_moe_add(out, x, S, K, H, F, idx, val, experts, mode, scratch);
}

#endif /* COLI_EXPERT_FFN_H */