Skip to content

Run frontier MoE models on hardware you already own β€” pure C, zero deps, experts streamed from disk. Tiny engine, immense model. 🐦

C
10.1 KB
Raw Blame
/* ============================================================================
 * test_gsgemv.c β€” standalone unit tests for the group-scaled int8 GEMV
 * (gsgemv.h: matmul_q_gs) that qwen36 runs for every expert matmul.
 * No model, no weights: links the SAME gsgemv.h the engine links.
 *
 * gsgemv.h has three tiers: AVX2/FMA (row-interleaved), SSE4.1 (row-
 * interleaved, routed through sse41_kernels.h) and scalar. The AVX2 and
 * scalar tiers must keep their exact sequence of float operations -- float
 * addition is not associative, and this engine's token stream is required
 * to stay byte-identical to the reference. reference_gs_gemv below is a
 * VERBATIM copy of the kernel as it shipped before either the AVX2
 * row-interleave or the SSE4.1 tier existed, and the AVX2/scalar properties
 * compare the live kernel against it with memcmp on raw float bits. The
 * SSE4.1 tier has no pre-existing byte-identical output to match -- its
 * reduction tree is a genuinely different shape -- so it alone is checked
 * by max relative+absolute error instead.
 *
 * P1 EXPERT SHAPES β€” the two shapes the engine actually runs (I=2048,O=512
 *    for gate/up and I=512,O=2048 for down, gs=64) match.
 * P2 ROW TAIL β€” O not a multiple of the unroll width still matches, so the
 *    tail rows cannot quietly accumulate in a different order.
 * P3 SCALAR PATH β€” gs not a multiple of 32 (AVX2) / 16 (SSE4.1) takes the
 *    scalar fallback. The dispatch is a runtime property of gs, so that
 *    path needs the same gate.
 * P4 PARTIAL GROUP β€” I not a multiple of gs leaves a short final group.
 * P5 SHORT FINAL GROUP β€” a final group of fewer than 16 (AVX2) / 8 (SSE4.1)
 *    elements. Both vector paths step through a group in fixed-width chunks
 *    and DROP such a remainder. That is pre-existing, dormant-in-this-model
 *    behaviour (I is 2048 or 512 and gs is 64, so the remainder is always
 *    zero), but it is pinned here so any future restructure reproduces it
 *    rather than silently correcting it -- a correction would change output,
 *    which is exactly what byte-identical tiers must not do. The SSE4.1
 *    tier's own tail-drop width (<8, half the AVX2 tier's <16) means
 *    reference_gs_gemv needs its OWN SSE4.1-shaped branch below: on an
 *    SSE4.1-only build, without that branch the reference would fall through
 *    to the undropped scalar path and P5 could never pass.
 * Exit 0 = all pass.
 * ==========================================================================*/
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <stdint.h>
#include <math.h>
#if defined(__AVX2__) && defined(__FMA__)
#include <immintrin.h>
#endif

#include "../gsgemv.h"

static int fails = 0;
#define CHECK(cond, msg) do { \
    if (cond) { printf("  ok   %s\n", msg); } \
    else { printf("  FAIL %s\n", msg); fails++; } \
} while (0)

/* Deterministic across libc implementations, unlike rand(). */
static uint32_t rng_state = 0x9E3779B9u;
static uint32_t rng_next(void) { rng_state = rng_state * 1664525u + 1013904223u; return rng_state; }
static float rng_f32(void) { return (float)((int32_t)(rng_next() >> 8) - 8388608) / 8388608.0f; }

/* ---------------------------------------------------------------------------
 * VERBATIM copy of matmul_q_gs as it shipped before the SSE4.1 tier was
 * added. Do not tidy, reformat or "improve" this: its value is that it is
 * the old arithmetic, character for character. If this drifts, the test
 * proves nothing.
 * ------------------------------------------------------------------------- */
static void reference_gs_gemv(float *y, const float *x, const int8_t *q, const float *scale,
                              int I, int O, int gs) {
    int ng = (I + gs - 1) / gs;
#if defined(__AVX2__) && defined(__FMA__)
    if ((gs & 31) == 0) {
        #pragma omp parallel for schedule(static) if(O >= 256)
        for (int o = 0; o < O; o++) {
            const int8_t *w = q + (int64_t)o * I;
            const float *sc = scale + (int64_t)o * ng;
            float acc = 0.f;
            for (int gi = 0; gi < ng; gi++) {
                __m256 a0 = _mm256_setzero_ps(), a1 = _mm256_setzero_ps();
                int base = gi * gs, end = base + gs; if (end > I) end = I;
                for (int i = base; i + 16 <= end; i += 16) {
                    __m128i b0 = _mm_loadu_si128((const __m128i*)(w + i));
                    a0 = _mm256_fmadd_ps(_mm256_loadu_ps(x+i),   _mm256_cvtepi32_ps(_mm256_cvtepi8_epi32(b0)), a0);
                    a1 = _mm256_fmadd_ps(_mm256_loadu_ps(x+i+8), _mm256_cvtepi32_ps(_mm256_cvtepi8_epi32(_mm_srli_si128(b0,8))), a1);
                }
                a0 = _mm256_add_ps(a0, a1);
                __m128 s = _mm_add_ps(_mm256_castps256_ps128(a0), _mm256_extractf128_ps(a0,1));
                s = _mm_add_ps(s, _mm_movehl_ps(s,s));
                s = _mm_add_ss(s, _mm_shuffle_ps(s,s,1));
                /* fmaf, not `acc += x * sc[gi]`: gsgemv.h's real AVX2 tier
                 * (gs_group_sum + fmaf) fuses this step explicitly, so an
                 * implicit multiply-add here only matches it when the compiler
                 * happens to auto-contract -- true at -O2/-O3 with the default
                 * -ffp-contract=fast, false at -O1 or with contraction off
                 * (confirmed: -O3 -ffp-contract=off reproduces the mismatch).
                 * Matching the explicit fmaf makes the comparison exact
                 * regardless of optimisation flags, instead of accidentally so. */
                acc = fmaf(_mm_cvtss_f32(s), sc[gi], acc);
            }
            y[o] = acc;
        }
        return;
    }
#elif defined(__SSE4_1__)
    /* Mirrors the SSE4.1 kernel's intentional <8 tail-drop so P5 pins that
     * behaviour instead of comparing against a reference that always sums
     * the full remainder. */
    if ((gs & 15) == 0) {
        #pragma omp parallel for schedule(static) if(O >= 256)
        for (int o = 0; o < O; o++) {
            const int8_t *w = q + (int64_t)o * I;
            const float *sc = scale + (int64_t)o * ng;
            float acc = 0.f;
            for (int gi = 0; gi < ng; gi++) {
                int base = gi * gs, end = base + gs; if (end > I) end = I;
                float part = 0.f;
                for (int i = base; i + 8 <= end; i += 8)
                    for (int k = 0; k < 8; k++) part += x[i+k] * (float)w[i+k];
                acc += part * sc[gi];
            }
            y[o] = acc;
        }
        return;
    }
#endif
    #pragma omp parallel for schedule(static) if(O >= 256)
    for (int o = 0; o < O; o++) {
        const int8_t *w = q + (int64_t)o * I;
        const float *sc = scale + (int64_t)o * ng;
        float acc = 0.f;
        for (int gi = 0; gi < ng; gi++) {
            int base = gi * gs, end = base + gs; if (end > I) end = I;
            float part = 0.f;
            for (int i = base; i < end; i++) part += x[i] * (float)w[i];
            acc += part * sc[gi];
        }
        y[o] = acc;
    }
}

/* Fills one random case, runs both kernels, compares the results.
 * The AVX2 and scalar tiers stay exact: memcmp on raw float bits, per the
 * engine's byte-identical requirement. The SSE4.1 tier's tree reduction is
 * not bit-reproducible against the scalar reference -- float addition is
 * not associative -- so that tier alone is checked by max relative error. */
static int rows_match(int I, int O, int gs) {
    int ng = (I + gs - 1) / gs;
    float  *x   = malloc((size_t)I * sizeof *x);
    int8_t *q   = malloc((size_t)O * I);
    float  *sc  = malloc((size_t)O * ng * sizeof *sc);
    float  *y   = malloc((size_t)O * sizeof *y);
    float  *ref = malloc((size_t)O * sizeof *ref);
    if (!x || !q || !sc || !y || !ref) { fprintf(stderr, "out of memory\n"); exit(2); }

    for (int i = 0; i < I; i++) x[i] = rng_f32();
    for (int64_t i = 0; i < (int64_t)O * I; i++) q[i] = (int8_t)(rng_next() >> 24);
    for (int i = 0; i < O * ng; i++) sc[i] = rng_f32() * 0.01f;

    /* Poison both outputs differently: a kernel that skips a row must not pass
     * by leaving stale bytes that happen to agree. */
    memset(y, 0x5A, (size_t)O * sizeof *y);
    memset(ref, 0xA5, (size_t)O * sizeof *ref);

    reference_gs_gemv(ref, x, q, sc, I, O, gs);
    matmul_q_gs(y, x, q, sc, I, O, gs);

#if defined(__SSE4_1__) && !(defined(__AVX2__) && defined(__FMA__))
    /* No FMA and a different tree reduction than the scalar reference: pure
     * relative error blows up on near-zero-ref rows (cancellation), so use a
     * combined absolute+relative bound instead. */
    int equal = 1;
    for (int o = 0; o < O; o++) {
        float diff = fabsf(y[o] - ref[o]);
        if (diff > 1e-5f + 1e-4f * fabsf(ref[o])) { equal = 0; break; }
    }
#else
    int equal = memcmp(y, ref, (size_t)O * sizeof *y) == 0;
#endif
    free(x); free(q); free(sc); free(y); free(ref);
    return equal;
}

int main(void) {
    printf("test_gsgemv: group-scaled int8 GEMV matches its reference\n");

    /* ---- P1: the shapes the engine actually runs ------------------------- */
    CHECK(rows_match(2048, 512, 64),  "P1 gate/up shape (I=2048, O=512, gs=64) matches");
    CHECK(rows_match(512, 2048, 64),  "P1 down shape (I=512, O=2048, gs=64) matches");

    /* ---- P2: O not a multiple of the unroll width ------------------------ */
    CHECK(rows_match(2048, 515, 64),  "P2 O=515 (tail rows) matches");
    CHECK(rows_match(2048, 6, 64),    "P2 O=6 (fewer rows than one unrolled block) matches");
    CHECK(rows_match(2048, 1, 64),    "P2 O=1 (single row) matches");

    /* ---- P3: gs not a multiple of 32/16 takes the scalar fallback -------- */
    CHECK(rows_match(2048, 512, 48),  "P3 gs=48 (scalar fallback path) matches");
    CHECK(rows_match(2000, 517, 48),  "P3 scalar fallback with ragged I and O matches");

    /* ---- P4: I not a multiple of gs leaves a short final group ----------- */
    CHECK(rows_match(2000, 512, 64),  "P4 I=2000 (final group of 16) matches");

    /* ---- P5: final group under the vector width, dropped by AVX2/SSE4.1 -- */
    CHECK(rows_match(1990, 512, 64),  "P5 I=1990 (final group of 6, dropped) reproduces the old result");

    printf("\n%s (%d failure%s)\n", fails ? "TEST FAIL" : "ALL PASS", fails, fails == 1 ? "" : "s");
    return fails ? 1 : 0;
}