Skip to content

Run frontier MoE models on hardware you already own β€” pure C, zero deps, experts streamed from disk. Tiny engine, immense model. 🐦

C
8.0 KB
Raw Blame
/* ============================================================================
 * test_qgemv.c β€” standalone unit tests for the plain (non-group-scaled) int8
 * GEMV (qgemv.h: matmul_q) that qwen36 runs for lm_head and any non-expert
 * quantized matmul. No model, no weights: links the SAME qgemv.h the engine
 * links.
 *
 * qgemv.h has an AVX2/FMA tier, an SSE4.1 tier (routed through
 * sse41_kernels.h) and a scalar tier (plus an ARM NEON/IDOT tier, dead code
 * on this x86 test build). The AVX2 and scalar tiers must keep their exact
 * sequence of float operations β€” float addition is not associative, and this
 * engine's token stream is required to stay byte-identical to the reference.
 * reference_q_gemv below is a VERBATIM copy of the kernel as it shipped
 * before the SSE4.1 tier was added, and the AVX2/scalar properties compare
 * the live kernel against it with memcmp on raw float bits. The SSE4.1 tier
 * has no pre-existing byte-identical output to match β€” its reduction tree is
 * a genuinely different shape β€” so it alone is checked by max relative error
 * instead.
 *
 * P1 DENSE SHAPES β€” the two shapes the engine actually runs (I=2048,O=512 and
 *    I=512,O=2048) are bit-identical.
 * P2 ROW COUNT EDGE CASES β€” O values around and below the OpenMP
 *    parallel-for threshold (O=515, O=6, O=1) are bit-identical. matmul_q has
 *    no row-interleaving, so this only exercises the scheduling boundary,
 *    not an unroll tail.
 * P4 I TAIL β€” I not a multiple of the unroll width (32 for AVX2, 16 for
 *    SSE4.1) leaves a final chunk that the unconditional scalar tail loop
 *    must still fold in correctly.
 * P5 SHORT I TAIL β€” a final chunk of just a few elements. Unlike
 *    matmul_q_gs's AVX2 tier, matmul_q's tail loop never drops a remainder,
 *    so this must match exactly, not reproduce a dropped-value quirk.
 * Exit 0 = all pass.
 * ==========================================================================*/
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <stdint.h>
#include <math.h>
#if defined(__AVX2__) && defined(__FMA__)
#include <immintrin.h>
#endif

#include "../qgemv.h"

static int fails = 0;
#define CHECK(cond, msg) do { \
    if (cond) { printf("  ok   %s\n", msg); } \
    else { printf("  FAIL %s\n", msg); fails++; } \
} while (0)

/* Deterministic across libc implementations, unlike rand(). */
static uint32_t rng_state = 0x9E3779B9u;
static uint32_t rng_next(void) { rng_state = rng_state * 1664525u + 1013904223u; return rng_state; }
static float rng_f32(void) { return (float)((int32_t)(rng_next() >> 8) - 8388608) / 8388608.0f; }

/* ---------------------------------------------------------------------------
 * VERBATIM copy of matmul_q as it shipped before the SSE4.1 tier was added.
 * Do not tidy, reformat or "improve" this: its value is that it is the old
 * arithmetic, character for character. If this drifts, the test proves nothing.
 * ------------------------------------------------------------------------- */
static void reference_q_gemv(float *y, const float *x, const int8_t *q, const float *scale, int I, int O) {
#if defined(__AVX2__) && defined(__FMA__)
    #pragma omp parallel for schedule(static) if(O >= 256)
    for (int o = 0; o < O; o++) {
        const int8_t *w = q + (int64_t)o * I;
        __m256 a0 = _mm256_setzero_ps(), a1 = _mm256_setzero_ps();
        __m256 a2 = _mm256_setzero_ps(), a3 = _mm256_setzero_ps();
        int i = 0;
        for (; i + 32 <= I; i += 32) {
            __m128i b0 = _mm_loadu_si128((const __m128i*)(w + i));
            __m128i b1 = _mm_loadu_si128((const __m128i*)(w + i + 16));
            a0 = _mm256_fmadd_ps(_mm256_loadu_ps(x+i),    _mm256_cvtepi32_ps(_mm256_cvtepi8_epi32(b0)), a0);
            a1 = _mm256_fmadd_ps(_mm256_loadu_ps(x+i+8),  _mm256_cvtepi32_ps(_mm256_cvtepi8_epi32(_mm_srli_si128(b0,8))), a1);
            a2 = _mm256_fmadd_ps(_mm256_loadu_ps(x+i+16), _mm256_cvtepi32_ps(_mm256_cvtepi8_epi32(b1)), a2);
            a3 = _mm256_fmadd_ps(_mm256_loadu_ps(x+i+24), _mm256_cvtepi32_ps(_mm256_cvtepi8_epi32(_mm_srli_si128(b1,8))), a3);
        }
        a0 = _mm256_add_ps(_mm256_add_ps(a0,a1), _mm256_add_ps(a2,a3));
        __m128 s = _mm_add_ps(_mm256_castps256_ps128(a0), _mm256_extractf128_ps(a0,1));
        s = _mm_add_ps(s, _mm_movehl_ps(s,s));
        s = _mm_add_ss(s, _mm_shuffle_ps(s,s,1));
        float acc = _mm_cvtss_f32(s);
        for (; i < I; i++) acc += x[i] * (float)w[i];
        y[o] = acc * scale[o];
    }
#else
    #pragma omp parallel for schedule(static)
    for (int o = 0; o < O; o++) {
        const int8_t *w = q + (int64_t)o * I;
        float acc = 0.f;
        for (int i = 0; i < I; i++) acc += x[i] * (float)w[i];
        y[o] = acc * scale[o];
    }
#endif
}

/* Fills one random case, runs both kernels, compares the results.
 * The AVX2 and scalar tiers stay exact: memcmp on raw float bits, per the
 * engine's byte-identical requirement. The SSE4.1 tier's tree reduction is
 * not bit-reproducible against the scalar reference -- float addition is
 * not associative -- so that tier alone is checked by max relative error. */
static int rows_match(int I, int O) {
    float  *x   = malloc((size_t)I * sizeof *x);
    int8_t *q   = malloc((size_t)O * I);
    float  *sc  = malloc((size_t)O * sizeof *sc);
    float  *y   = malloc((size_t)O * sizeof *y);
    float  *ref = malloc((size_t)O * sizeof *ref);
    if (!x || !q || !sc || !y || !ref) { fprintf(stderr, "out of memory\n"); exit(2); }

    for (int i = 0; i < I; i++) x[i] = rng_f32();
    for (int64_t i = 0; i < (int64_t)O * I; i++) q[i] = (int8_t)(rng_next() >> 24);
    for (int o = 0; o < O; o++) sc[o] = rng_f32() * 0.01f;

    /* Poison both outputs differently: a kernel that skips a row must not pass
     * by leaving stale bytes that happen to agree. */
    memset(y, 0x5A, (size_t)O * sizeof *y);
    memset(ref, 0xA5, (size_t)O * sizeof *ref);

    reference_q_gemv(ref, x, q, sc, I, O);
    matmul_q(y, x, q, sc, I, O);

#if defined(__SSE4_1__) && !(defined(__AVX2__) && defined(__FMA__))
    /* Combined absolute+relative tolerance (numpy allclose-style), not pure
     * relative error: a tree-reduced SSE4.1 sum and a sequential scalar sum
     * disagree by float-rounding noise, and when a reference output happens
     * to land near zero that noise alone blows the relative error past any
     * pure-relative threshold. Measured worst-case noise across all shapes
     * below is ~4.2e-6 absolute; the 1e-5 floor gives headroom above that
     * without loosening the check for the far more common non-near-zero
     * outputs, where 1e-4 relative still rules. */
    int equal = 1;
    for (int o = 0; o < O; o++) {
        float diff  = fabsf(y[o] - ref[o]);
        float denom = fabsf(ref[o]);
        if (diff > 1e-5f + 1e-4f * denom) { equal = 0; break; }
    }
#else
    int equal = memcmp(y, ref, (size_t)O * sizeof *y) == 0;
#endif
    free(x); free(q); free(sc); free(y); free(ref);
    return equal;
}

int main(void) {
    printf("test_qgemv: plain int8 GEMV matches its reference\n");

    /* ---- P1: the shapes the engine actually runs ------------------------- */
    CHECK(rows_match(2048, 512),  "P1 lm_head-like shape (I=2048, O=512) matches");
    CHECK(rows_match(512, 2048),  "P1 dense shape (I=512, O=2048) matches");

    /* ---- P2: row count around the OpenMP parallel-for threshold ---------- */
    CHECK(rows_match(2048, 515),  "P2 O=515 (just over the parallel threshold) matches");
    CHECK(rows_match(2048, 6),    "P2 O=6 (few rows) matches");
    CHECK(rows_match(2048, 1),    "P2 O=1 (single row) matches");

    /* ---- P4: I not a multiple of the unroll width leaves a final chunk --- */
    CHECK(rows_match(2000, 512),  "P4 I=2000 (final chunk of 16) matches");

    /* ---- P5: a short final chunk, folded in by the unconditional tail ---- */
    CHECK(rows_match(1990, 512),  "P5 I=1990 (final chunk of 6) matches");
    CHECK(rows_match(1, 4),       "P5 I=1 (no full unrolled chunk at all) matches");

    printf("\n%s (%d failure%s)\n", fails ? "TEST FAIL" : "ALL PASS", fails, fails == 1 ? "" : "s");
    return fails ? 1 : 0;
}