Skip to content

Run frontier MoE models on hardware you already own β€” pure C, zero deps, experts streamed from disk. Tiny engine, immense model. 🐦

C
81.3 KB
Raw Blame
/* quant.h β€” quantized matmul kernels (header-only, all functions static).
 * Multi-architecture SIMD: AVX2 / AVX-512 / AVX-VNNI / ARM NEON / NEON-SDOT /
 * NEON-i8mm / POWER VSX.  Pure compute β€” no Model or QT dependency. */
#ifndef COLI_QUANT_H
#define COLI_QUANT_H

#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <math.h>
#include <stdint.h>

#ifdef _OPENMP
#include <omp.h>
#endif

#include "idot.h"   /* SIMD prelude + the integer dot kernels, shared with qwen36 */

#if defined(__SSE4_1__)
#include "sse41_kernels.h"
#endif

/* ---- AVX-512 int4->float accumulator -------------------------------------- */
#if defined(__AVX512F__) && defined(__AVX512BW__)
static int g_i4_acc512=1;
static inline float dot_i4f_avx512(const uint8_t *w,const float *x,int I){
    const __m128i m4=_mm_set1_epi8(0x0F); const __m512i b8=_mm512_set1_epi32(8);
    __m512 acc0=_mm512_setzero_ps(),acc1=_mm512_setzero_ps(); int i=0;
    for(;i+32<=I;i+=32){ __m128i by=_mm_loadu_si128((const __m128i*)(w+(i>>1)));
        __m128i lo=_mm_and_si128(by,m4),hi=_mm_and_si128(_mm_srli_epi16(by,4),m4);
        __m128i n0=_mm_unpacklo_epi8(lo,hi),n1=_mm_unpackhi_epi8(lo,hi);
        __m512 w0=_mm512_cvtepi32_ps(_mm512_sub_epi32(_mm512_cvtepu8_epi32(n0),b8));
        __m512 w1=_mm512_cvtepi32_ps(_mm512_sub_epi32(_mm512_cvtepu8_epi32(n1),b8));
        acc0=_mm512_fmadd_ps(_mm512_loadu_ps(x+i),w0,acc0);
        acc1=_mm512_fmadd_ps(_mm512_loadu_ps(x+i+16),w1,acc1);
    }
    return _mm512_reduce_add_ps(_mm512_add_ps(acc0,acc1));
}
/* acc[0..I) += coef * dequant(int4 row) β€” the axpy twin of dot_i4f_avx512, for the
 * MLA absorption path (qt_addrow). Each acc[i] receives exactly ONE fma per call
 * (no cross-element accumulation), so this is bit-identical to the scalar loop
 * (gcc -O3 -ffp-contract already emits scalar fma there). Tail handled scalar. */
static inline void axpy_i4f_avx512(const uint8_t *w,float coef,float *acc,int I){
    const __m128i m4=_mm_set1_epi8(0x0F); const __m512i b8=_mm512_set1_epi32(8);
    const __m512 cv=_mm512_set1_ps(coef); int i=0;
    for(;i+32<=I;i+=32){ __m128i by=_mm_loadu_si128((const __m128i*)(w+(i>>1)));
        __m128i lo=_mm_and_si128(by,m4),hi=_mm_and_si128(_mm_srli_epi16(by,4),m4);
        __m128i n0=_mm_unpacklo_epi8(lo,hi),n1=_mm_unpackhi_epi8(lo,hi);
        __m512 w0=_mm512_cvtepi32_ps(_mm512_sub_epi32(_mm512_cvtepu8_epi32(n0),b8));
        __m512 w1=_mm512_cvtepi32_ps(_mm512_sub_epi32(_mm512_cvtepu8_epi32(n1),b8));
        _mm512_storeu_ps(acc+i,   _mm512_fmadd_ps(cv,w0,_mm512_loadu_ps(acc+i)));
        _mm512_storeu_ps(acc+i+16,_mm512_fmadd_ps(cv,w1,_mm512_loadu_ps(acc+i+16)));
    }
    for(;i+1<I;i+=2){ uint8_t b=w[i>>1]; acc[i]+=coef*(float)((int)(b&0xF)-8); acc[i+1]+=coef*(float)((int)(b>>4)-8); }
    if(i<I){ uint8_t b=w[i>>1]; acc[i]+=coef*(float)((int)(b&0xF)-8); }
}
static int i4_acc512_selftest(void){
    enum { N=224 }; uint8_t w[(N+1)/2]; float x[N];
    for(int i=0;i<N;i++){
        int q=((i*13+5)&15)-8;
        if(!(i&1)) w[i>>1]=(uint8_t)(q+8);
        else w[i>>1]|=(uint8_t)((q+8)<<4);
        x[i]=(float)(((i*29+7)%101)-50)/37.f;
    }
    for(int n=32;n<=N;n+=32){
        float ref=0; for(int i=0;i<n;i++) ref+=x[i]*(float)(((w[i>>1]>>((i&1)*4))&15)-8);
        float got=dot_i4f_avx512(w,x,n),tol=2e-5f*(1.f+fabsf(ref));
        if(fabsf(got-ref)>tol){ fprintf(stderr,"AVX512 i4 selftest n=%d: %.9g != %.9g\n",n,got,ref); return 0; }
    }
    return 1;
}
#endif

/* ---- y[S,O] = x[S,I] @ W^T, W[O,I] f32: matmul_f32.h ------------------- */
#include "matmul_f32.h"

/* ---- y[S,O] = x[S,I] @ W^T, W int8 per-row + scale[O] ------------------- */
static void matmul_q(float *y, const float *x, const int8_t *q, const float *scale, int S, int I, int O){
    #pragma omp parallel for schedule(static)
    for (int o=0;o<O;o++){ const int8_t *w=q+(int64_t)o*I; float sc=scale[o];
        for (int s=0;s<S;s++){ const float *xs=x+(int64_t)s*I; float a=0; int i=0;
#ifdef __AVX2__
            __m256 acc=_mm256_setzero_ps();
            for(;i+8<=I;i+=8){ __m256i wi=_mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*)(w+i)));
                acc=_mm256_fmadd_ps(_mm256_loadu_ps(xs+i), _mm256_cvtepi32_ps(wi), acc); }
            a=hsum256(acc);
#elif defined(__ARM_NEON)
            float32x4_t ac0=vdupq_n_f32(0), ac1=vdupq_n_f32(0);
            for(;i+8<=I;i+=8){ int16x8_t w16=vmovl_s8(vld1_s8(w+i));
                ac0=vfmaq_f32(ac0, vld1q_f32(xs+i),   vcvtq_f32_s32(vmovl_s16(vget_low_s16(w16))));
                ac1=vfmaq_f32(ac1, vld1q_f32(xs+i+4), vcvtq_f32_s32(vmovl_s16(vget_high_s16(w16)))); }
            a=vaddvq_f32(vaddq_f32(ac0,ac1));
#endif
            for(;i<I;i++) a+=xs[i]*(float)w[i]; y[(int64_t)s*O+o]=a*sc; } }
}

/* ---- y[S,O] = x[S,I] @ W^T, W int4 packed (2/byte) + scale[O] ------------ */
static void matmul_i4(float *y, const float *x, const uint8_t *q4, const float *scale, int S, int I, int O){
    int rb=(I+1)/2;
    #pragma omp parallel for schedule(static)
    for (int o=0;o<O;o++){ const uint8_t *w=q4+(int64_t)o*rb; float sc=scale[o];
        for (int s=0;s<S;s++){ const float *xs=x+(int64_t)s*I; float a=0; int i=0;
#if defined(__AVX512F__) && defined(__AVX512BW__)
            if(g_i4_acc512){ a=dot_i4f_avx512(w,xs,I); i=I&~31; }
            else {
#endif
#ifdef __AVX2__
            const __m128i m4=_mm_set1_epi8(0x0F); const __m256i b8=_mm256_set1_epi32(8);
            __m256 acc=_mm256_setzero_ps();
            for(;i+16<=I;i+=16){ __m128i by=_mm_loadl_epi64((const __m128i*)(w+(i>>1)));
                __m128i lo=_mm_and_si128(by,m4), hi=_mm_and_si128(_mm_srli_epi16(by,4),m4);
                __m128i nib=_mm_unpacklo_epi8(lo,hi);
                __m256 w0=_mm256_cvtepi32_ps(_mm256_sub_epi32(_mm256_cvtepu8_epi32(nib),b8));
                __m256 w1=_mm256_cvtepi32_ps(_mm256_sub_epi32(_mm256_cvtepu8_epi32(_mm_srli_si128(nib,8)),b8));
                acc=_mm256_fmadd_ps(_mm256_loadu_ps(xs+i),   w0, acc);
                acc=_mm256_fmadd_ps(_mm256_loadu_ps(xs+i+8), w1, acc); }
            a=hsum256(acc);
#elif defined(__ARM_NEON)
            const uint8x8_t m4=vdup_n_u8(0x0F); const int8x8_t b8=vdup_n_s8(8);
            float32x4_t ac0=vdupq_n_f32(0), ac1=vdupq_n_f32(0);
            for(;i+16<=I;i+=16){ uint8x8_t by=vld1_u8(w+(i>>1));
                uint8x8x2_t z=vzip_u8(vand_u8(by,m4), vshr_n_u8(by,4));
                int16x8_t w0=vmovl_s8(vsub_s8(vreinterpret_s8_u8(z.val[0]),b8));
                int16x8_t w1=vmovl_s8(vsub_s8(vreinterpret_s8_u8(z.val[1]),b8));
                ac0=vfmaq_f32(ac0, vld1q_f32(xs+i),    vcvtq_f32_s32(vmovl_s16(vget_low_s16(w0))));
                ac1=vfmaq_f32(ac1, vld1q_f32(xs+i+4),  vcvtq_f32_s32(vmovl_s16(vget_high_s16(w0))));
                ac0=vfmaq_f32(ac0, vld1q_f32(xs+i+8),  vcvtq_f32_s32(vmovl_s16(vget_low_s16(w1))));
                ac1=vfmaq_f32(ac1, vld1q_f32(xs+i+12), vcvtq_f32_s32(vmovl_s16(vget_high_s16(w1)))); }
            a=vaddvq_f32(vaddq_f32(ac0,ac1));
#endif
#if defined(__AVX512F__) && defined(__AVX512BW__)
            }
#endif
            for(;i+1<I;i+=2){ uint8_t byte=w[i>>1]; int lo=(int)(byte&0xF)-8, hi=(int)(byte>>4)-8;
                a += xs[i]*(float)lo + xs[i+1]*(float)hi; }
            if(i<I){ uint8_t byte=w[i>>1]; int lo=(int)(byte&0xF)-8; a += xs[i]*(float)lo; }
            y[(int64_t)s*O+o]=a*sc; } }
}

/* ---- y[S,O] = x[S,I] @ W^T, W int4 packed + per-GROUP scales (fmt=4) ----- */
static void matmul_i4_grouped(float *y, const float *x, const uint8_t *q4, const float *scale,
                              int S, int I, int O, int gs){
    int rb=(I+1)/2; int ng=(I+gs-1)/gs;
    int o0=0;
#if defined(__SSE4_1__) && !defined(__AVX2__)
    /* Even group sizes keep every group start on a low-nibble boundary. */
    if(!(gs&1)){
        o0=O&~3;
        if(o0) matmul_i4_grouped_sse41_rows4(y,x,q4,scale,S,I,O,gs,rb,ng,o0);
        if(o0==O) return;
    }
#endif
    #pragma omp parallel for schedule(static)
    for(int o=o0;o<O;o++){
        const uint8_t *w=q4+(int64_t)o*rb;
        const float *scl=scale+(int64_t)o*ng;
        for(int s=0;s<S;s++){
            const float *xs=x+(int64_t)s*I; float a=0;
            for(int g=0; g*gs<I; g++){
                int base=g*gs; int glen=gs; if(base+glen>I) glen=I-base;
                float sc=scl[g];
                int i=base;
#ifdef __AVX2__
                const __m128i m4=_mm_set1_epi8(0x0F); const __m256i b8=_mm256_set1_epi32(8);
                __m256 acc=_mm256_setzero_ps();
                for(; i+16<=base+glen; i+=16){ __m128i by=_mm_loadl_epi64((const __m128i*)(w+(i>>1)));
                    __m128i lo=_mm_and_si128(by,m4),hi=_mm_and_si128(_mm_srli_epi16(by,4),m4);
                    __m128i nib=_mm_unpacklo_epi8(lo,hi);
                    __m256 w0=_mm256_cvtepi32_ps(_mm256_sub_epi32(_mm256_cvtepu8_epi32(nib),b8));
                    __m256 w1=_mm256_cvtepi32_ps(_mm256_sub_epi32(_mm256_cvtepu8_epi32(_mm_srli_si128(nib,8)),b8));
                    acc=_mm256_fmadd_ps(_mm256_loadu_ps(xs+i),   w0, acc);
                    acc=_mm256_fmadd_ps(_mm256_loadu_ps(xs+i+8), w1, acc); }
                /* Pinned as an fma in the SOURCE. With the default
                 * -ffp-contract=fast the compiler may fuse this multiply-add
                 * (one rounding) or not (two), so the same source produced
                 * different bits depending on the flag -- see the PR for a
                 * reproduction. That made every bit-exactness gate, including
                 * the glm_tiny token oracle, depend on build flags rather than
                 * on the code. */
                a=fmaf(hsum256(acc),sc,a);
#endif
                for(; i<base+glen; i+=2){
                    if(i+1<base+glen){ uint8_t byte=w[i>>1];
                        a+=(xs[i]*(float)((int)(byte&0xF)-8)+xs[i+1]*(float)((int)(byte>>4)-8))*sc; }
                    else { uint8_t byte=w[i>>1]; a+=xs[i]*(float)((int)(byte&0xF)-8)*sc; }
                }
            }
            y[(int64_t)s*O+o]=a;
        }
    }
}

/* ---- fused gate+up: one OMP dispatch for both matrices -------------------- */
static void matmul_i4_pair(float *yg, float *yu, const float *x,
                           const uint8_t *qg, const float *sg,
                           const uint8_t *qu, const float *su, int I, int O){
    int rb=(I+1)/2;
    #pragma omp parallel for schedule(static)
    for(int z=0;z<2*O;z++){
        int o=z<O?z:z-O; const uint8_t *w=(z<O?qg:qu)+(int64_t)o*rb;
        float a=0; int i=0;
#if defined(__AVX512F__) && defined(__AVX512BW__)
        if(g_i4_acc512){ a=dot_i4f_avx512(w,x,I); i=I&~31; }
        else {
#endif
#ifdef __AVX2__
        const __m128i m4=_mm_set1_epi8(0x0F); const __m256i b8=_mm256_set1_epi32(8);
        __m256 acc=_mm256_setzero_ps();
        for(;i+16<=I;i+=16){ __m128i by=_mm_loadl_epi64((const __m128i*)(w+(i>>1)));
            __m128i lo=_mm_and_si128(by,m4),hi=_mm_and_si128(_mm_srli_epi16(by,4),m4);
            __m128i nib=_mm_unpacklo_epi8(lo,hi);
            __m256 w0=_mm256_cvtepi32_ps(_mm256_sub_epi32(_mm256_cvtepu8_epi32(nib),b8));
            __m256 w1=_mm256_cvtepi32_ps(_mm256_sub_epi32(_mm256_cvtepu8_epi32(_mm_srli_si128(nib,8)),b8));
            acc=_mm256_fmadd_ps(_mm256_loadu_ps(x+i),w0,acc);
            acc=_mm256_fmadd_ps(_mm256_loadu_ps(x+i+8),w1,acc); }
        a=hsum256(acc);
#elif defined(__ARM_NEON)
        const uint8x8_t m4=vdup_n_u8(0x0F); const int8x8_t b8=vdup_n_s8(8);
        float32x4_t ac0=vdupq_n_f32(0),ac1=vdupq_n_f32(0);
        for(;i+16<=I;i+=16){ uint8x8_t by=vld1_u8(w+(i>>1));
            uint8x8x2_t n=vzip_u8(vand_u8(by,m4),vshr_n_u8(by,4));
            int16x8_t w0=vmovl_s8(vsub_s8(vreinterpret_s8_u8(n.val[0]),b8));
            int16x8_t w1=vmovl_s8(vsub_s8(vreinterpret_s8_u8(n.val[1]),b8));
            ac0=vfmaq_f32(ac0,vld1q_f32(x+i),vcvtq_f32_s32(vmovl_s16(vget_low_s16(w0))));
            ac1=vfmaq_f32(ac1,vld1q_f32(x+i+4),vcvtq_f32_s32(vmovl_s16(vget_high_s16(w0))));
            ac0=vfmaq_f32(ac0,vld1q_f32(x+i+8),vcvtq_f32_s32(vmovl_s16(vget_low_s16(w1))));
            ac1=vfmaq_f32(ac1,vld1q_f32(x+i+12),vcvtq_f32_s32(vmovl_s16(vget_high_s16(w1)))); }
        a=vaddvq_f32(vaddq_f32(ac0,ac1));
#endif
#if defined(__AVX512F__) && defined(__AVX512BW__)
        }
#endif
        for(;i+1<I;i+=2){ uint8_t b=w[i>>1]; a+=x[i]*(float)((b&15)-8)+x[i+1]*(float)((b>>4)-8); }
        if(i<I) a+=x[i]*(float)((w[i>>1]&15)-8);
        (z<O?yg:yu)[o]=a*(z<O?sg:su)[o];
    }
}

/* ---- y[S,O] = x[S,I] @ W^T, W int2 packed (4/byte) + scale[O] ------------ */
static void matmul_i2(float *y, const float *x, const uint8_t *q2, const float *scale, int S, int I, int O){
    int rb=(I+3)/4;
    #pragma omp parallel for schedule(static)
    for (int o=0;o<O;o++){ const uint8_t *w=q2+(int64_t)o*rb; float sc=scale[o];
        for (int s=0;s<S;s++){ const float *xs=x+(int64_t)s*I; float a=0; int i=0;
#ifdef __AVX2__
            const __m128i m2=_mm_set1_epi8(0x03); const __m256i b2=_mm256_set1_epi32(2);
            __m256 acc=_mm256_setzero_ps();
            for(;i+16<=I;i+=16){ __m128i by=_mm_cvtsi32_si128(*(const int*)(w+(i>>2)));
                __m128i p0=_mm_and_si128(by,m2), p1=_mm_and_si128(_mm_srli_epi16(by,2),m2);
                __m128i p2=_mm_and_si128(_mm_srli_epi16(by,4),m2), p3=_mm_and_si128(_mm_srli_epi16(by,6),m2);
                __m128i lo=_mm_unpacklo_epi8(p0,p1), hi=_mm_unpacklo_epi8(p2,p3);
                __m128i nib=_mm_unpacklo_epi16(lo,hi);
                __m256 w0=_mm256_cvtepi32_ps(_mm256_sub_epi32(_mm256_cvtepu8_epi32(nib),b2));
                __m256 w1=_mm256_cvtepi32_ps(_mm256_sub_epi32(_mm256_cvtepu8_epi32(_mm_srli_si128(nib,8)),b2));
                acc=_mm256_fmadd_ps(_mm256_loadu_ps(xs+i),   w0, acc);
                acc=_mm256_fmadd_ps(_mm256_loadu_ps(xs+i+8), w1, acc); }
            a=hsum256(acc);
#elif defined(__ARM_NEON)
            const uint8x8_t m2v=vdup_n_u8(3); const int8x8_t b2v=vdup_n_s8(2);
            float32x4_t ac0=vdupq_n_f32(0), ac1=vdupq_n_f32(0);
            for(;i+16<=I;i+=16){ uint32_t wd; memcpy(&wd, w+(i>>2), 4);
                uint8x8_t by=vreinterpret_u8_u32(vdup_n_u32(wd));
                uint8x8x2_t z01=vzip_u8(vand_u8(by,m2v),              vand_u8(vshr_n_u8(by,2),m2v));
                uint8x8x2_t z23=vzip_u8(vand_u8(vshr_n_u8(by,4),m2v), vshr_n_u8(by,6));
                uint16x4x2_t zz=vzip_u16(vreinterpret_u16_u8(z01.val[0]), vreinterpret_u16_u8(z23.val[0]));
                int16x8_t w0=vmovl_s8(vsub_s8(vreinterpret_s8_u16(zz.val[0]),b2v));
                int16x8_t w1=vmovl_s8(vsub_s8(vreinterpret_s8_u16(zz.val[1]),b2v));
                ac0=vfmaq_f32(ac0, vld1q_f32(xs+i),    vcvtq_f32_s32(vmovl_s16(vget_low_s16(w0))));
                ac1=vfmaq_f32(ac1, vld1q_f32(xs+i+4),  vcvtq_f32_s32(vmovl_s16(vget_high_s16(w0))));
                ac0=vfmaq_f32(ac0, vld1q_f32(xs+i+8),  vcvtq_f32_s32(vmovl_s16(vget_low_s16(w1))));
                ac1=vfmaq_f32(ac1, vld1q_f32(xs+i+12), vcvtq_f32_s32(vmovl_s16(vget_high_s16(w1)))); }
            a=vaddvq_f32(vaddq_f32(ac0,ac1));
#endif
            for(;i<I;i++){ uint8_t byte=w[i>>2]; int sh=(i&3)*2; a += xs[i]*(float)((int)((byte>>sh)&3)-2); }
            y[(int64_t)s*O+o]=a*sc; } }
}

/* ---- int3-g64 (fmt=5): 3-bit weights with ONE f32 scale per 64-input group -
 * Per group: 16B low plane (2 bits/val, int2 layout) + 8B high plane (1 bit/val),
 * values in [-4,3] stored v+4. 3.5 bits/weight effective β€” the quality/size point
 * the #132 OLMoE ablation measured BEATING per-row int4. */
#define I3_GROUP 64
#define I3_GBYTES 24                     /* 16B low plane + 8B high plane per group */
static inline int64_t i3_groups(int I){ return ((int64_t)I + I3_GROUP - 1) / I3_GROUP; }
static inline int64_t i3_rowbytes(int I){ return i3_groups(I) * I3_GBYTES; }

#if defined(__AVX512F__) && defined(__AVX512BW__)
static int g_i3_avx512=1;
/* one full 64-value group -> f32 partial. Relies on immintrin.h arriving via the
 * __AVX2__-gated include above (AVX512F implies AVX2 on clang/gcc/MSVC), same as
 * dot_i4f_avx512. */
static inline float dot_i3g64_avx512(const uint8_t *lo, const uint8_t *hi, const float *x){
    const __m128i m2=_mm_set1_epi8(3); const __m512i c4=_mm512_set1_epi8(4);
    __m128i by=_mm_loadu_si128((const __m128i*)lo);
    __m128i p0=_mm_and_si128(by,m2),                   p1=_mm_and_si128(_mm_srli_epi16(by,2),m2);
    __m128i p2=_mm_and_si128(_mm_srli_epi16(by,4),m2), p3=_mm_and_si128(_mm_srli_epi16(by,6),m2);
    __m128i l01=_mm_unpacklo_epi8(p0,p1), h01=_mm_unpackhi_epi8(p0,p1);
    __m128i l23=_mm_unpacklo_epi8(p2,p3), h23=_mm_unpackhi_epi8(p2,p3);
    __m512i lov=_mm512_inserti32x4(_mm512_inserti32x4(_mm512_inserti32x4(
        _mm512_castsi128_si512(_mm_unpacklo_epi16(l01,l23)),
        _mm_unpackhi_epi16(l01,l23),1),
        _mm_unpacklo_epi16(h01,h23),2),
        _mm_unpackhi_epi16(h01,h23),3);            /* byte k = low 2 bits of value k */
    uint64_t hb; memcpy(&hb,hi,8);                 /* mask bit k = high bit of value k */
    __m512i wq=_mm512_sub_epi8(_mm512_mask_add_epi8(lov,(__mmask64)hb,lov,c4),c4); /* [-4,3] in order */
    __m512 ac0=_mm512_setzero_ps(), ac1=_mm512_setzero_ps();
    ac0=_mm512_fmadd_ps(_mm512_loadu_ps(x),    _mm512_cvtepi32_ps(_mm512_cvtepi8_epi32(_mm512_castsi512_si128(wq))),      ac0);
    ac1=_mm512_fmadd_ps(_mm512_loadu_ps(x+16), _mm512_cvtepi32_ps(_mm512_cvtepi8_epi32(_mm512_extracti32x4_epi32(wq,1))), ac1);
    ac0=_mm512_fmadd_ps(_mm512_loadu_ps(x+32), _mm512_cvtepi32_ps(_mm512_cvtepi8_epi32(_mm512_extracti32x4_epi32(wq,2))), ac0);
    ac1=_mm512_fmadd_ps(_mm512_loadu_ps(x+48), _mm512_cvtepi32_ps(_mm512_cvtepi8_epi32(_mm512_extracti32x4_epi32(wq,3))), ac1);
    return _mm512_reduce_add_ps(_mm512_add_ps(ac0,ac1));
}
static int i3_avx512_selftest(void){
    /* fixed group, asymmetric in every lane: pseudo-random 3-bit values with
     * distinct nonzero integer activations. All terms and partials are small
     * integers (exact in f32 under ANY summation order), so the compare is
     * exact β€” any lane permutation, bias error or plane mix-up shifts the sum. */
    uint8_t lo[16]={0}, hi[8]={0}; float x[I3_GROUP]; double ref=0;
    uint64_t r=0x9E3779B97F4A7C15ull;
    for(int k=0;k<I3_GROUP;k++){
        r^=r<<13; r^=r>>7; r^=r<<17;
        unsigned u=(unsigned)(r&7);
        lo[k>>2]|=(uint8_t)((u&3)<<((k&3)*2));
        hi[k>>3]|=(uint8_t)((u>>2)<<(k&7));
        x[k]=(k&1)?-(float)(k+1):(float)(k+1);
        ref+=(double)x[k]*((int)u-4);
    }
    float got=dot_i3g64_avx512(lo,hi,x);
    if(got!=(float)ref){ fprintf(stderr,"AVX512 i3 selftest: %.9g != %.9g\n",got,ref); return 0; }
    return 1;
}
#endif

/* Dequant-on-use with PER-GROUP scale. Exact f32 path only (no IDOT in v1: int8
 * activations don't compose with per-group accumulation without a kernel
 * restructure β€” follow-up). NEON: low plane = matmul_i2's unpack, high plane
 * expanded via vtst on bit masks. AVX2: same unpack, high plane via bit-test
 * against per-lane masks. AVX-512(F+BW): same unpack at 128-bit, high plane
 * loaded as a __mmask64 (bit k = value k) driving a masked +4; one full group
 * per iteration (dot_i3g64_avx512; I3_AVX512=0 falls back to scalar). All
 * vector arms reorder fma WITHIN a group only; the per-group partial is
 * scaled by scale[g] and added to the row accumulator in scalar order,
 * exactly like the scalar loop. */
static void matmul_i3(float *y, const float *x, const uint8_t *q3, const float *scale, int S, int I, int O){
    int64_t ng=i3_groups(I), rb=i3_rowbytes(I);
    #pragma omp parallel for schedule(static)
    for(int o=0;o<O;o++){
        const uint8_t *wrow=q3+(int64_t)o*rb;
        const float *srow=scale+(int64_t)o*ng;
        for(int s=0;s<S;s++){
            const float *xs=x+(int64_t)s*I;
            float acc=0;
            for(int64_t g=0; g<ng; g++){
                const uint8_t *lo=wrow+g*I3_GBYTES, *hi=lo+16;
                int base=(int)(g*I3_GROUP), n = I-base < I3_GROUP ? I-base : I3_GROUP;
                float a=0; int k=0;
#if defined(__AVX512F__) && defined(__AVX512BW__)
                if(g_i3_avx512 && n==I3_GROUP){ a=dot_i3g64_avx512(lo,hi,xs+base); k=I3_GROUP; }
#elif defined(__ARM_NEON)
                if(n==I3_GROUP){
                    const uint8x8_t m2v=vdup_n_u8(3); const int8x16_t b4q=vdupq_n_s8(4);
                    const uint8x16_t bitm={1,2,4,8,16,32,64,128,1,2,4,8,16,32,64,128};
                    const uint8x16_t fourq=vdupq_n_u8(4);
                    float32x4_t ac0=vdupq_n_f32(0), ac1=vdupq_n_f32(0);
                    for(;k+16<=I3_GROUP;k+=16){
                        uint32_t wd; memcpy(&wd, lo+(k>>2), 4);            /* 4 bytes = 16 low-plane values */
                        uint8x8_t by=vreinterpret_u8_u32(vdup_n_u32(wd));
                        uint8x8x2_t z01=vzip_u8(vand_u8(by,m2v),              vand_u8(vshr_n_u8(by,2),m2v));
                        uint8x8x2_t z23=vzip_u8(vand_u8(vshr_n_u8(by,4),m2v), vshr_n_u8(by,6));
                        uint16x4x2_t zz=vzip_u16(vreinterpret_u16_u8(z01.val[0]), vreinterpret_u16_u8(z23.val[0]));
                        uint8x16_t lov=vcombine_u8(vreinterpret_u8_u16(zz.val[0]), vreinterpret_u8_u16(zz.val[1]));
                        uint8x16_t hv=vcombine_u8(vdup_n_u8(hi[k>>3]), vdup_n_u8(hi[(k>>3)+1]));
                        uint8x16_t hb=vandq_u8(vtstq_u8(hv,bitm), fourq);   /* 4 where high bit set */
                        int8x16_t wq=vsubq_s8(vreinterpretq_s8_u8(vaddq_u8(lov,hb)), b4q); /* [-4,3] in order */
                        int16x8_t w0=vmovl_s8(vget_low_s8(wq)), w1=vmovl_s8(vget_high_s8(wq));
                        ac0=vfmaq_f32(ac0, vld1q_f32(xs+base+k),    vcvtq_f32_s32(vmovl_s16(vget_low_s16(w0))));
                        ac1=vfmaq_f32(ac1, vld1q_f32(xs+base+k+4),  vcvtq_f32_s32(vmovl_s16(vget_high_s16(w0))));
                        ac0=vfmaq_f32(ac0, vld1q_f32(xs+base+k+8),  vcvtq_f32_s32(vmovl_s16(vget_low_s16(w1))));
                        ac1=vfmaq_f32(ac1, vld1q_f32(xs+base+k+12), vcvtq_f32_s32(vmovl_s16(vget_high_s16(w1))));
                    }
                    a=vaddvq_f32(vaddq_f32(ac0,ac1));
                }
#elif defined(__AVX2__)
                if(n==I3_GROUP){
                    const __m128i m2=_mm_set1_epi8(0x03);
                    const __m128i bsel=_mm_set_epi8(1,1,1,1,1,1,1,1, 0,0,0,0,0,0,0,0);
                    const __m128i bitm=_mm_set_epi8((char)128,64,32,16,8,4,2,1,(char)128,64,32,16,8,4,2,1);
                    const __m128i four8=_mm_set1_epi8(4);
                    const __m256i b4=_mm256_set1_epi32(4);
                    __m256 ac0=_mm256_setzero_ps(), ac1=_mm256_setzero_ps();
                    for(;k+16<=I3_GROUP;k+=16){
                        __m128i by=_mm_cvtsi32_si128(*(const int*)(lo+(k>>2)));  /* 4 bytes = 16 low-plane values */
                        __m128i p0=_mm_and_si128(by,m2), p1=_mm_and_si128(_mm_srli_epi16(by,2),m2);
                        __m128i p2=_mm_and_si128(_mm_srli_epi16(by,4),m2), p3=_mm_and_si128(_mm_srli_epi16(by,6),m2);
                        __m128i l01=_mm_unpacklo_epi8(p0,p1), h23=_mm_unpacklo_epi8(p2,p3);
                        __m128i lov=_mm_unpacklo_epi16(l01,h23);                  /* 16 vals in order */
                        __m128i hv=_mm_shuffle_epi8(_mm_cvtsi32_si128(hi[k>>3]|(hi[(k>>3)+1]<<8)),bsel);
                        __m128i hb=_mm_and_si128(_mm_cmpeq_epi8(_mm_and_si128(hv,bitm),bitm),four8); /* 4 where high bit set */
                        __m128i u=_mm_add_epi8(lov,hb);                           /* [0,7] stored v+4 */
                        __m256 w0=_mm256_cvtepi32_ps(_mm256_sub_epi32(_mm256_cvtepu8_epi32(u),b4));
                        __m256 w1=_mm256_cvtepi32_ps(_mm256_sub_epi32(_mm256_cvtepu8_epi32(_mm_srli_si128(u,8)),b4));
                        ac0=_mm256_fmadd_ps(_mm256_loadu_ps(xs+base+k),  w0,ac0);
                        ac1=_mm256_fmadd_ps(_mm256_loadu_ps(xs+base+k+8),w1,ac1);
                    }
                    a=hsum256(_mm256_add_ps(ac0,ac1));
                }
#endif
                for(;k<n;k++){
                    unsigned u=((lo[k>>2]>>((k&3)*2))&3) | (((hi[k>>3]>>(k&7))&1)<<2);
                    a += xs[base+k]*(float)((int)u-4);
                }
                acc += a*srow[g];
            }
            y[(int64_t)s*O+o]=acc;
        }
    }
}

/* ---- fmt=8: native FP8-e4m3 passthrough (Z.ai GLM-5.2-FP8 read path) ------
 * PUBLIC ordinal. This format was minted fmt=6 during original development,
 * then re-tagged fmt=100 (PRIVATE ORDINAL BLOCK, see colibri.c's QT struct
 * comment) after #465 (E8/IQ3, above) claimed ordinal 6 upstream out from
 * under it; graduated to fmt=7 when the maintainer assigned that ordinal on
 * #524; renumbered a second time to fmt=8 after #705 merged into dev claiming
 * fmt=7 for MXFP4 (Kimi K3 Vulkan tier, backend_vulkan.c) while this PR was
 * still open. Both renumbers were pure retags, per the private-block
 * convention's own "find-and-replace, zero on-disk impact" promise: nothing
 * on disk carries the ordinal. See qt_resolve_fmt in colibri.c ("THE DESIGN
 * LANDMINE") for the disambiguation this needs against BOTH fmt=1 (int8) and
 * fmt=6 (E8/IQ3).
 *
 * fmt=8 weight bytes are O*I raw e4m3 bytes -- byte-identical to int8 (fmt=1).
 * What makes this a DIFFERENT format is the scale: one f32 per 128x128 BLOCK of
 * the [O,I] weight matrix (shape [ceil(O/128),ceil(I/128)]), not one f32 per
 * output row. Dequant is w[o,i] = e4m3_decode(byte) * scale[o/128, i/128]
 * (MULTIPLY, not divide) -- mirrors tools/convert_fp8_to_int4.py's dequant()
 * exactly, which is the authoritative reference for how Z.ai's checkpoints read
 * on the source side. This f32 encoding is a declared PROPERTY of the format,
 * not the only one it can carry -- DeepSeek-V4 ships this identical weight
 * geometry with UE8M0 (1-byte, power-of-two) block scales instead; qt_resolve_fmt
 * recognizes that byte signature and refuses it BY NAME rather than
 * misreading it as f32 (UE8M0 decode itself is not implemented in this build).
 *
 * 256-entry decode LUT, cross-checked byte-for-byte against PyTorch's
 * torch.float8_e4m3fn (the exact dtype safetensors reports for these shards):
 * sign(1) exp(4,bias=7) mant(3), subnormal at exp==0, and the OCP E4M3-FN
 * convention that exp==0xF is NOT reserved for infinity -- only mant==0x7 at
 * exp==0xF is NaN (max finite is exp=0xF,mant=0x6 -> 448). A static compile-time
 * table (not a lazily-initialized one) is used deliberately: matmul_fp8 below
 * runs inside #pragma omp parallel for, and a lazy-init global would race under
 * concurrent first use.
 *
 * NaN POLICY (decided, documented per the build spec): a NaN byte (0x7F/0xFF)
 * decodes to a real IEEE NaN and is left to propagate through the dot product,
 * exactly like any other source of a NaN weight (fmt=0/f32 tensors are never
 * scrubbed either). The engine already has a dedicated, TESTED safety net for
 * NaN reaching the sampler (argmax_v/dist_build, see tests/test_logit_nan.c:
 * "degrade + diagnose, never silently corrupt") -- fmt=8 relies on that existing
 * downstream net rather than adding a second, redundant weight-level scrub. */
static const float E4M3_LUT[256] = {
    0x0.0p+0f,0x1.0000000000000p-9f,0x1.0000000000000p-8f,0x1.8000000000000p-8f,0x1.0000000000000p-7f,0x1.4000000000000p-7f,0x1.8000000000000p-7f,0x1.c000000000000p-7f,
    0x1.0000000000000p-6f,0x1.2000000000000p-6f,0x1.4000000000000p-6f,0x1.6000000000000p-6f,0x1.8000000000000p-6f,0x1.a000000000000p-6f,0x1.c000000000000p-6f,0x1.e000000000000p-6f,
    0x1.0000000000000p-5f,0x1.2000000000000p-5f,0x1.4000000000000p-5f,0x1.6000000000000p-5f,0x1.8000000000000p-5f,0x1.a000000000000p-5f,0x1.c000000000000p-5f,0x1.e000000000000p-5f,
    0x1.0000000000000p-4f,0x1.2000000000000p-4f,0x1.4000000000000p-4f,0x1.6000000000000p-4f,0x1.8000000000000p-4f,0x1.a000000000000p-4f,0x1.c000000000000p-4f,0x1.e000000000000p-4f,
    0x1.0000000000000p-3f,0x1.2000000000000p-3f,0x1.4000000000000p-3f,0x1.6000000000000p-3f,0x1.8000000000000p-3f,0x1.a000000000000p-3f,0x1.c000000000000p-3f,0x1.e000000000000p-3f,
    0x1.0000000000000p-2f,0x1.2000000000000p-2f,0x1.4000000000000p-2f,0x1.6000000000000p-2f,0x1.8000000000000p-2f,0x1.a000000000000p-2f,0x1.c000000000000p-2f,0x1.e000000000000p-2f,
    0x1.0000000000000p-1f,0x1.2000000000000p-1f,0x1.4000000000000p-1f,0x1.6000000000000p-1f,0x1.8000000000000p-1f,0x1.a000000000000p-1f,0x1.c000000000000p-1f,0x1.e000000000000p-1f,
    0x1.0000000000000p+0f,0x1.2000000000000p+0f,0x1.4000000000000p+0f,0x1.6000000000000p+0f,0x1.8000000000000p+0f,0x1.a000000000000p+0f,0x1.c000000000000p+0f,0x1.e000000000000p+0f,
    0x1.0000000000000p+1f,0x1.2000000000000p+1f,0x1.4000000000000p+1f,0x1.6000000000000p+1f,0x1.8000000000000p+1f,0x1.a000000000000p+1f,0x1.c000000000000p+1f,0x1.e000000000000p+1f,
    0x1.0000000000000p+2f,0x1.2000000000000p+2f,0x1.4000000000000p+2f,0x1.6000000000000p+2f,0x1.8000000000000p+2f,0x1.a000000000000p+2f,0x1.c000000000000p+2f,0x1.e000000000000p+2f,
    0x1.0000000000000p+3f,0x1.2000000000000p+3f,0x1.4000000000000p+3f,0x1.6000000000000p+3f,0x1.8000000000000p+3f,0x1.a000000000000p+3f,0x1.c000000000000p+3f,0x1.e000000000000p+3f,
    0x1.0000000000000p+4f,0x1.2000000000000p+4f,0x1.4000000000000p+4f,0x1.6000000000000p+4f,0x1.8000000000000p+4f,0x1.a000000000000p+4f,0x1.c000000000000p+4f,0x1.e000000000000p+4f,
    0x1.0000000000000p+5f,0x1.2000000000000p+5f,0x1.4000000000000p+5f,0x1.6000000000000p+5f,0x1.8000000000000p+5f,0x1.a000000000000p+5f,0x1.c000000000000p+5f,0x1.e000000000000p+5f,
    0x1.0000000000000p+6f,0x1.2000000000000p+6f,0x1.4000000000000p+6f,0x1.6000000000000p+6f,0x1.8000000000000p+6f,0x1.a000000000000p+6f,0x1.c000000000000p+6f,0x1.e000000000000p+6f,
    0x1.0000000000000p+7f,0x1.2000000000000p+7f,0x1.4000000000000p+7f,0x1.6000000000000p+7f,0x1.8000000000000p+7f,0x1.a000000000000p+7f,0x1.c000000000000p+7f,0x1.e000000000000p+7f,
    0x1.0000000000000p+8f,0x1.2000000000000p+8f,0x1.4000000000000p+8f,0x1.6000000000000p+8f,0x1.8000000000000p+8f,0x1.a000000000000p+8f,0x1.c000000000000p+8f,       NAN,
     -0x0.0p+0f,-0x1.0000000000000p-9f,-0x1.0000000000000p-8f,-0x1.8000000000000p-8f,-0x1.0000000000000p-7f,-0x1.4000000000000p-7f,-0x1.8000000000000p-7f,-0x1.c000000000000p-7f,
    -0x1.0000000000000p-6f,-0x1.2000000000000p-6f,-0x1.4000000000000p-6f,-0x1.6000000000000p-6f,-0x1.8000000000000p-6f,-0x1.a000000000000p-6f,-0x1.c000000000000p-6f,-0x1.e000000000000p-6f,
    -0x1.0000000000000p-5f,-0x1.2000000000000p-5f,-0x1.4000000000000p-5f,-0x1.6000000000000p-5f,-0x1.8000000000000p-5f,-0x1.a000000000000p-5f,-0x1.c000000000000p-5f,-0x1.e000000000000p-5f,
    -0x1.0000000000000p-4f,-0x1.2000000000000p-4f,-0x1.4000000000000p-4f,-0x1.6000000000000p-4f,-0x1.8000000000000p-4f,-0x1.a000000000000p-4f,-0x1.c000000000000p-4f,-0x1.e000000000000p-4f,
    -0x1.0000000000000p-3f,-0x1.2000000000000p-3f,-0x1.4000000000000p-3f,-0x1.6000000000000p-3f,-0x1.8000000000000p-3f,-0x1.a000000000000p-3f,-0x1.c000000000000p-3f,-0x1.e000000000000p-3f,
    -0x1.0000000000000p-2f,-0x1.2000000000000p-2f,-0x1.4000000000000p-2f,-0x1.6000000000000p-2f,-0x1.8000000000000p-2f,-0x1.a000000000000p-2f,-0x1.c000000000000p-2f,-0x1.e000000000000p-2f,
    -0x1.0000000000000p-1f,-0x1.2000000000000p-1f,-0x1.4000000000000p-1f,-0x1.6000000000000p-1f,-0x1.8000000000000p-1f,-0x1.a000000000000p-1f,-0x1.c000000000000p-1f,-0x1.e000000000000p-1f,
    -0x1.0000000000000p+0f,-0x1.2000000000000p+0f,-0x1.4000000000000p+0f,-0x1.6000000000000p+0f,-0x1.8000000000000p+0f,-0x1.a000000000000p+0f,-0x1.c000000000000p+0f,-0x1.e000000000000p+0f,
    -0x1.0000000000000p+1f,-0x1.2000000000000p+1f,-0x1.4000000000000p+1f,-0x1.6000000000000p+1f,-0x1.8000000000000p+1f,-0x1.a000000000000p+1f,-0x1.c000000000000p+1f,-0x1.e000000000000p+1f,
    -0x1.0000000000000p+2f,-0x1.2000000000000p+2f,-0x1.4000000000000p+2f,-0x1.6000000000000p+2f,-0x1.8000000000000p+2f,-0x1.a000000000000p+2f,-0x1.c000000000000p+2f,-0x1.e000000000000p+2f,
    -0x1.0000000000000p+3f,-0x1.2000000000000p+3f,-0x1.4000000000000p+3f,-0x1.6000000000000p+3f,-0x1.8000000000000p+3f,-0x1.a000000000000p+3f,-0x1.c000000000000p+3f,-0x1.e000000000000p+3f,
    -0x1.0000000000000p+4f,-0x1.2000000000000p+4f,-0x1.4000000000000p+4f,-0x1.6000000000000p+4f,-0x1.8000000000000p+4f,-0x1.a000000000000p+4f,-0x1.c000000000000p+4f,-0x1.e000000000000p+4f,
    -0x1.0000000000000p+5f,-0x1.2000000000000p+5f,-0x1.4000000000000p+5f,-0x1.6000000000000p+5f,-0x1.8000000000000p+5f,-0x1.a000000000000p+5f,-0x1.c000000000000p+5f,-0x1.e000000000000p+5f,
    -0x1.0000000000000p+6f,-0x1.2000000000000p+6f,-0x1.4000000000000p+6f,-0x1.6000000000000p+6f,-0x1.8000000000000p+6f,-0x1.a000000000000p+6f,-0x1.c000000000000p+6f,-0x1.e000000000000p+6f,
    -0x1.0000000000000p+7f,-0x1.2000000000000p+7f,-0x1.4000000000000p+7f,-0x1.6000000000000p+7f,-0x1.8000000000000p+7f,-0x1.a000000000000p+7f,-0x1.c000000000000p+7f,-0x1.e000000000000p+7f,
    -0x1.0000000000000p+8f,-0x1.2000000000000p+8f,-0x1.4000000000000p+8f,-0x1.6000000000000p+8f,-0x1.8000000000000p+8f,-0x1.a000000000000p+8f,-0x1.c000000000000p+8f,       NAN,
};
static inline float e4m3_decode(uint8_t b){ return E4M3_LUT[b]; }

#ifdef __AVX2__
/* Eight e4m3 bytes to eight floats without touching a lookup table.
 *
 * A float wants sign, exponent, mantissa in that order, and e4m3 already has
 * them in that order: place the byte's seven low bits at bit 20 and its sign at
 * bit 31, and the result is the right number with the wrong exponent bias,
 * 127 where e4m3 means 7. One multiply by 2^120 fixes it, and it fixes the
 * subnormals too, because scaling a subnormal by a power of two into the normal
 * range is exact.
 *
 * Verified byte for byte against quant.h's E4M3_LUT: 254 of the 256 encodings
 * come out identical. The two that do not are 0x7F and 0xFF, e4m3's NaNs, which
 * this would turn into +-480 -- a corrupt weight quietly becoming a plausible
 * one. They are blended back, because a NaN that propagates is how you find out.
 */
static inline __m256 e4m3_decode8(const uint8_t *p) {
    __m256i b = _mm256_cvtepu8_epi32(_mm_loadl_epi64((const __m128i *)p));
    __m256i low = _mm256_and_si256(b, _mm256_set1_epi32(0x7F));
    __m256i bits = _mm256_or_si256(
        _mm256_slli_epi32(_mm256_and_si256(b, _mm256_set1_epi32(0x80)), 24),
        _mm256_slli_epi32(low, 20));
    __m256 value = _mm256_mul_ps(_mm256_castsi256_ps(bits), _mm256_set1_ps(0x1p120f));
    __m256i nan = _mm256_cmpeq_epi32(low, _mm256_set1_epi32(0x7F));
    return _mm256_blendv_ps(value, _mm256_set1_ps(NAN), _mm256_castsi256_ps(nan));
}

/* Eight bf16 into eight floats: bf16 IS the top half of a float, so the whole
 * conversion is a shift. Exact for every encoding, NaNs and infinities
 * included, which is why there is no table to disagree with. */
static inline __m256 bf16_decode8(const uint16_t *p) {
    __m256i w = _mm256_cvtepu16_epi32(_mm_loadu_si128((const __m128i *)p));
    return _mm256_castsi256_ps(_mm256_slli_epi32(w, 16));
}
#endif

/* FP8_BLOCK / fp8_nblk moved to fp8_format.h so the CUDA backend shares the
 * same named constant instead of restating 128 as literals (see that header's
 * comment for the drift hazard this closes). */
#include "fp8_format.h"

/* y[S,O] = x[S,I] @ W^T, W raw e4m3 bytes (byte-identical layout to fmt=1) +
 * per-128x128-BLOCK f32 scale [ceil(O/128),ceil(I/128)]. Scalar reference path
 * (no SIMD in v1 -- BW-bound like the Metal kernel, and this format's hot path
 * is the GPU one; a vectorized CPU kernel is future work if measured needed).
 * Mirrors matmul_i3's double-accumulate-across-groups / float-within-group
 * convention so cross-block cancellation doesn't cost precision unfairly. */
/* Two kernels, selected by compiler, because the faster one is only
   BIT-EXACT under clang.

   The four-accumulator form is algebraically identical to the one-accumulator
   loop - same operands, same column order - so any difference comes purely
   from the compiler contracting multiply-adds differently in the four-chain
   shape, which drifts results by ~1 ulp.  That would break the byte-exact
   contract tests/test_qwen38_native_weights.c pins against its independent
   reference, and a one-ulp logit can flip an argmax in the token-exact gates.

   Under GCC that contraction cannot be controlled from source.  Measured on
   gcc 13.4, clean build per cell, against this kernel's own test:
     - #pragma GCC optimize ("fp-contract=off")     ignored (no-op)
     - #pragma GCC optimize ("-ffp-contract=off")   ignored (no-op)
     - __attribute__((optimize("-ffp-contract=off"))) ignored (no-op)
     - #pragma STDC FP_CONTRACT OFF                 unimplemented: GCC warns
                                                    "ignoring '#pragma STDC
                                                    FP_CONTRACT'"
   Only the command-line -ffp-contract=off works, and that is a global numerics
   decision this kernel has no business making for the whole project.  With no
   guard, the four-row form is exact on -march=znver3 and -march=x86-64-v3 but
   NOT on -march=haswell, so "it passed on my machine" is not evidence here.

   clang contracts the reference and this kernel identically, so the four-row
   form is bit-exact there: verified on arm64 (binary byte-identical to the
   one-row build) and on x86 clang at -march=haswell, znver3 and x86-64-v3.
   So clang gets the fast kernel and GCC keeps upstream's, which is exact on
   every arch tested. */
#if defined(__clang__)
static void matmul_fp8_scalar(float *y, const float *x, const uint8_t *q8, const float *bscale,
                              int S, int I, int O){
    int64_t nblkI = fp8_nblk(I);
    /* Four output rows per pass, each with its own accumulator.  Every row's
       addition sequence is identical to the one-row form - same operands, same
       column order - so results are bit-identical; only the interleaving of four
       independent dependency chains differs.  The gain is latency hiding plus one
       load of xs[i] feeding four rows.  A tail of fewer than four rows clamps the
       spare indices onto the last valid row and the stores are guarded, which
       avoids a separate remainder loop. */
    #pragma omp parallel for schedule(static)
    for(int o=0;o<O;o+=4){
        int o1=o+1<O ? o+1 : o, o2=o+2<O ? o+2 : o, o3=o+3<O ? o+3 : o;
        const uint8_t *w0=q8+(int64_t)o*I,  *w1=q8+(int64_t)o1*I;
        const uint8_t *w2=q8+(int64_t)o2*I, *w3=q8+(int64_t)o3*I;
        const float *scl0=bscale+((int64_t)o /FP8_BLOCK)*nblkI;
        const float *scl1=bscale+((int64_t)o1/FP8_BLOCK)*nblkI;
        const float *scl2=bscale+((int64_t)o2/FP8_BLOCK)*nblkI;
        const float *scl3=bscale+((int64_t)o3/FP8_BLOCK)*nblkI;
        for(int s=0;s<S;s++){
            const float *xs = x + (int64_t)s*I;
            double a0=0,a1=0,a2=0,a3=0;
            for(int64_t bi=0; bi*FP8_BLOCK<I; bi++){
                int base=(int)(bi*FP8_BLOCK); int blen=FP8_BLOCK; if(base+blen>I) blen=I-base;
                float acc0=0,acc1=0,acc2=0,acc3=0;
                for(int i=base;i<base+blen;i++){
                    float xv=xs[i];
#if defined(__aarch64__)
                    /* SVE review: explicit fma pins the chain FUSED regardless of
                       -ffp-contract or the auto-vectoriser (gcc on SVE lowers a
                       plain acc += w*x to unfused fadda and diverges from vfmaq). */
                    acc0 = __builtin_fmaf(e4m3_decode(w0[i]),xv,acc0);
                    acc1 = __builtin_fmaf(e4m3_decode(w1[i]),xv,acc1);
                    acc2 = __builtin_fmaf(e4m3_decode(w2[i]),xv,acc2);
                    acc3 = __builtin_fmaf(e4m3_decode(w3[i]),xv,acc3);
#else
                    acc0 += e4m3_decode(w0[i])*xv;
                    acc1 += e4m3_decode(w1[i])*xv;
                    acc2 += e4m3_decode(w2[i])*xv;
                    acc3 += e4m3_decode(w3[i])*xv;
#endif
                }
                a0 += (double)acc0*scl0[bi];
                a1 += (double)acc1*scl1[bi];
                a2 += (double)acc2*scl2[bi];
                a3 += (double)acc3*scl3[bi];
            }
            y[(int64_t)s*O+o]=(float)a0;
            if(o1!=o) y[(int64_t)s*O+o1]=(float)a1;
            if(o2!=o) y[(int64_t)s*O+o2]=(float)a2;
            if(o3!=o) y[(int64_t)s*O+o3]=(float)a3;
        }
    }
}
#else
/* GCC and everything else: upstream's one-row kernel, unchanged.  Exact on
   every -march tested; see the note above for why it is not simply replaced. */
static void matmul_fp8_scalar(float *y, const float *x, const uint8_t *q8, const float *bscale,
                              int S, int I, int O){
    int64_t nblkI = fp8_nblk(I);
    #pragma omp parallel for schedule(static)
    for(int o=0;o<O;o++){
        const uint8_t *w = q8 + (int64_t)o*I;
        int64_t blkO = o / FP8_BLOCK;
        const float *scl = bscale + blkO*nblkI;
        for(int s=0;s<S;s++){
            const float *xs = x + (int64_t)s*I;
            double a=0;
            for(int64_t bi=0; bi*FP8_BLOCK<I; bi++){
                int base=(int)(bi*FP8_BLOCK); int blen=FP8_BLOCK; if(base+blen>I) blen=I-base;
                float sc=scl[bi]; float acc=0;
#if defined(__aarch64__)
                /* see the SVE note in the clang arm above */
                for(int i=base;i<base+blen;i++) acc = __builtin_fmaf(e4m3_decode(w[i]),xs[i],acc);
#else
                for(int i=base;i<base+blen;i++) acc += e4m3_decode(w[i])*xs[i];
#endif
                a += (double)acc*sc;
            }
            y[(int64_t)s*O+o]=(float)a;
        }
    }
}
#endif

#ifdef __ARM_NEON
/* Batched (prefill) NEON arm: 16 output rows x 4 tokens per pass.  Each block's
   16 rows are decoded once into a column-major tile; each row's per-block sum is
   then one vfmaq chain over i = base..base+blen-1 from zero -- the same fused
   multiply-adds, in the same order, that clang (-ffp-contract=on) and GCC
   (-ffp-contract=fast, the arm64 default) emit for the scalar form -- and the
   double-precision block combine is unchanged, so the output is bit-identical to
   matmul_fp8_scalar (test_fp8_passthrough checks it).  A 16-row tile never
   straddles a FP8_BLOCK row block.  Rows past O are zero-padded and never
   stored; token lanes past S clamp onto the last token and are never summed.
   S < 4 stays scalar: one token cannot amortise the tile decode. */
static void matmul_fp8_neon(float *y, const float *x, const uint8_t *q8, const float *bscale,
                            int S, int I, int O){
    int64_t nblkI = fp8_nblk(I);
    #pragma omp parallel for schedule(static)
    for(int o=0;o<O;o+=16){
        int rn = O-o<16 ? O-o : 16;
        const uint8_t *w = q8 + (int64_t)o*I;
        const float *scl = bscale + ((int64_t)o/FP8_BLOCK)*nblkI;
        float wt[FP8_BLOCK*16];                       /* wt[i*16+r] */
        double a[64*16];
        for(int s0=0;s0<S;s0+=64){
            int sn = S-s0<64 ? S-s0 : 64;
            memset(a,0,sizeof(double)*16*sn);
            for(int64_t bi=0; bi*FP8_BLOCK<I; bi++){
                int base=(int)(bi*FP8_BLOCK), blen=I-base<FP8_BLOCK ? I-base : FP8_BLOCK;
                double sc=scl[bi];
                for(int r=0;r<16;r++) for(int i=0;i<blen;i++)
                    wt[i*16+r] = r<rn ? e4m3_decode(w[(int64_t)r*I+base+i]) : 0.f;
                for(int s=0;s<sn;s+=4){
                    const float *xt[4];
                    float32x4_t c[4][4];
                    for(int t=0;t<4;t++){
                        xt[t] = x + (int64_t)(s0+(s+t<sn ? s+t : sn-1))*I + base;
                        for(int k=0;k<4;k++) c[t][k]=vdupq_n_f32(0.f);
                    }
                    for(int i=0;i<blen;i++){
                        const float *p = wt+i*16;
                        float32x4_t w0=vld1q_f32(p),w1=vld1q_f32(p+4),w2=vld1q_f32(p+8),w3=vld1q_f32(p+12);
                        for(int t=0;t<4;t++){
                            float32x4_t xv=vdupq_n_f32(xt[t][i]);
                            c[t][0]=vfmaq_f32(c[t][0],w0,xv); c[t][1]=vfmaq_f32(c[t][1],w1,xv);
                            c[t][2]=vfmaq_f32(c[t][2],w2,xv); c[t][3]=vfmaq_f32(c[t][3],w3,xv);
                        }
                    }
                    for(int t=0;t<4 && s+t<sn;t++){
                        float acc[16]; double *as=a+(s+t)*16;
                        for(int k=0;k<4;k++) vst1q_f32(acc+4*k,c[t][k]);
                        for(int r=0;r<16;r++) as[r] += (double)acc[r]*sc;
                    }
                }
            }
            for(int s=0;s<sn;s++) for(int r=0;r<rn;r++) y[(int64_t)(s0+s)*O+o+r]=(float)a[s*16+r];
        }
    }
}
#endif

static void matmul_fp8(float *y, const float *x, const uint8_t *q8, const float *bscale,
                       int S, int I, int O){
#ifdef __ARM_NEON
    if(S>=4){ matmul_fp8_neon(y,x,q8,bscale,S,I,O); return; }
#endif
    matmul_fp8_scalar(y,x,q8,bscale,S,I,O);
}


/* f32 planare (fmt=2): stesso ordine di accumulazione di matmul_i4 (sequenza
 * 0..63 in blocchi di 8 lane, stessa lane per elemento) -> bit-identico. */
static void matmul_i4p(float *y, const float *x, const uint8_t *q4, const float *scale,
                       int S, int I, int O){
#ifdef __AVX2__
    int rb=(I+1)/2;
    #pragma omp parallel for schedule(static)
    for(int o=0;o<O;o++){
        const uint8_t *w=q4+(int64_t)o*rb; float sc=scale[o];
        for(int s=0;s<S;s++){
            const float *xs=x+(int64_t)s*I;
            const __m256i m4=_mm256_set1_epi32(0xF), b8=_mm256_set1_epi32(8);
            __m256 acc=_mm256_setzero_ps();
            int i=0;
            for(;i+64<=I;i+=64){
                __m128i lo0=_mm_loadu_si128((const __m128i*)(w+(i>>1)));      /* byte 0..15 */
                __m128i lo1=_mm_loadu_si128((const __m128i*)(w+(i>>1)+16));   /* byte 16..31 */
                /* elementi 0..31: nibble bassi in ordine */
                __m256i e0=_mm256_and_si256(_mm256_cvtepu8_epi32(lo0),m4);
                __m256i e1=_mm256_and_si256(_mm256_cvtepu8_epi32(_mm_srli_si128(lo0,8)),m4);
                __m256i e2=_mm256_and_si256(_mm256_cvtepu8_epi32(lo1),m4);
                __m256i e3=_mm256_and_si256(_mm256_cvtepu8_epi32(_mm_srli_si128(lo1,8)),m4);
                acc=_mm256_fmadd_ps(_mm256_loadu_ps(xs+i),
                                    _mm256_cvtepi32_ps(_mm256_sub_epi32(e0,b8)),acc);
                acc=_mm256_fmadd_ps(_mm256_loadu_ps(xs+i+8),
                                    _mm256_cvtepi32_ps(_mm256_sub_epi32(e1,b8)),acc);
                acc=_mm256_fmadd_ps(_mm256_loadu_ps(xs+i+16),
                                    _mm256_cvtepi32_ps(_mm256_sub_epi32(e2,b8)),acc);
                acc=_mm256_fmadd_ps(_mm256_loadu_ps(xs+i+24),
                                    _mm256_cvtepi32_ps(_mm256_sub_epi32(e3,b8)),acc);
                /* elementi 32..63: nibble alti in ordine */
                __m256i h0=_mm256_srli_epi32(_mm256_cvtepu8_epi32(lo0),4);
                __m256i h1=_mm256_srli_epi32(_mm256_cvtepu8_epi32(_mm_srli_si128(lo0,8)),4);
                __m256i h2=_mm256_srli_epi32(_mm256_cvtepu8_epi32(lo1),4);
                __m256i h3=_mm256_srli_epi32(_mm256_cvtepu8_epi32(_mm_srli_si128(lo1,8)),4);
                acc=_mm256_fmadd_ps(_mm256_loadu_ps(xs+i+32),
                                    _mm256_cvtepi32_ps(_mm256_sub_epi32(_mm256_and_si256(h0,m4),b8)),acc);
                acc=_mm256_fmadd_ps(_mm256_loadu_ps(xs+i+40),
                                    _mm256_cvtepi32_ps(_mm256_sub_epi32(_mm256_and_si256(h1,m4),b8)),acc);
                acc=_mm256_fmadd_ps(_mm256_loadu_ps(xs+i+48),
                                    _mm256_cvtepi32_ps(_mm256_sub_epi32(_mm256_and_si256(h2,m4),b8)),acc);
                acc=_mm256_fmadd_ps(_mm256_loadu_ps(xs+i+56),
                                    _mm256_cvtepi32_ps(_mm256_sub_epi32(_mm256_and_si256(h3,m4),b8)),acc);
            }
            /* coda: PRIMA il loop vettoriale a 16 del gemello (stesso acc,
             * stessa sequenza per-lane {8m+L}: nessuna cucitura), POI la coda
             * scalare identica. EN: tail = the pair kernel's 16-wide loop into
             * the same acc (seamless per-lane order), then its scalar tail. */
            {
                const __m128i m4p=_mm_set1_epi8(0x0F); const __m256i b8p=_mm256_set1_epi32(8);
                for(;i+16<=I;i+=16){ __m128i by=_mm_loadl_epi64((const __m128i*)(w+(i>>1)));
                    __m128i lo=_mm_and_si128(by,m4p), hi=_mm_and_si128(_mm_srli_epi16(by,4),m4p);
                    __m128i nib=_mm_unpacklo_epi8(lo,hi);
                    __m256 w0=_mm256_cvtepi32_ps(_mm256_sub_epi32(_mm256_cvtepu8_epi32(nib),b8p));
                    __m256 w1=_mm256_cvtepi32_ps(_mm256_sub_epi32(_mm256_cvtepu8_epi32(_mm_srli_si128(nib,8)),b8p));
                    acc=_mm256_fmadd_ps(_mm256_loadu_ps(xs+i),   w0, acc);
                    acc=_mm256_fmadd_ps(_mm256_loadu_ps(xs+i+8), w1, acc); }
            }
            float a=hsum256(acc);
            for(;i+1<I;i+=2){ uint8_t byte=w[i>>1]; int lo=(int)(byte&0xF)-8, hi=(int)(byte>>4)-8;
                a += xs[i]*(float)lo + xs[i+1]*(float)hi; }
            if(i<I){ uint8_t byte=w[i>>1]; int lo=(int)(byte&0xF)-8; a += xs[i]*(float)lo; }
            y[(int64_t)s*O+o]=a*sc;
        }
    }
#else
    /* scalare: blocchi planari + coda a coppie */
    int rb=(I+1)/2;
    #pragma omp parallel for schedule(static)
    for(int o=0;o<O;o++){
        const uint8_t *w=q4+(int64_t)o*rb; float sc=scale[o];
        for(int s=0;s<S;s++){
            const float *xs=x+(int64_t)s*I; float a=0; int i=0;
            for(;i+64<=I;i+=64){
                const uint8_t *blk=w+(i>>1);
                for(int k=0;k<32;k++){
                    a+=xs[i+k]*(float)((int)(blk[k]&0xF)-8);
                    a+=xs[i+k+32]*(float)((int)(blk[k]>>4)-8);
                }
            }
            for(;i<I;i+=2){
                uint8_t byte=w[i>>1];
                a+=xs[i]*(float)((int)(byte&0xF)-8);
                if(i+1<I) a+=xs[i+1]*(float)((int)(byte>>4)-8);
            }
            y[(int64_t)s*O+o]=a*sc;
        }
    }
#endif
}
/* ============================ fine blocco K1 ============================== */

/* ---- per-thread quantization scratch -------------------------------------- */
typedef struct { int8_t *xq; size_t xq_cap; float *sx; size_t sx_cap; } QScratch;
static _Thread_local QScratch g_qscratch;
static void quant_scratch(size_t xn, size_t sn, int8_t **xq, float **sx){
    if(xn>g_qscratch.xq_cap){
        int8_t *p=(int8_t*)realloc(g_qscratch.xq,xn);
        if(!p){ fprintf(stderr,"OOM quant scratch\n"); exit(1); }
        g_qscratch.xq=p; g_qscratch.xq_cap=xn;
    }
    if(sn>g_qscratch.sx_cap){
        float *p=(float*)realloc(g_qscratch.sx,sn*sizeof(float));
        if(!p){ fprintf(stderr,"OOM quant scales\n"); exit(1); }
        g_qscratch.sx=p; g_qscratch.sx_cap=sn;
    }
    *xq=g_qscratch.xq; *sx=g_qscratch.sx;
}

/* ---- f32 -> quantized packing --------------------------------------------- */
static void quantize_rows(const float *w, int8_t *q, float *scale, int O, int I, int bits){
    int qmax=(1<<(bits-1))-1;
    #pragma omp parallel for schedule(static)
    for(int o=0;o<O;o++){ const float *wr=w+(int64_t)o*I; float amax=0;
        for(int i=0;i<I;i++){ float a=fabsf(wr[i]); if(a>amax)amax=a; }
        float s=amax/qmax; if(s<1e-8f)s=1e-8f; scale[o]=s;
        int8_t *qr=q+(int64_t)o*I;
        for(int i=0;i<I;i++){ int v=(int)lrintf(wr[i]/s); if(v>qmax)v=qmax; if(v<-qmax-1)v=-qmax-1; qr[i]=(int8_t)v; }
    }
}
static void pack_int4(const float *w, uint8_t *q4, float *scale, int O, int I, int bits){
    int qmax=(1<<(bits-1))-1, rb=(I+1)/2;
    #pragma omp parallel for schedule(static)
    for(int o=0;o<O;o++){ const float *wr=w+(int64_t)o*I; float amax=0;
        for(int i=0;i<I;i++){ float a=fabsf(wr[i]); if(a>amax)amax=a; }
        float s=amax/qmax; if(s<1e-8f)s=1e-8f; scale[o]=s;
        uint8_t *qr=q4+(int64_t)o*rb;
        for(int i=0;i<I;i+=2){
            int v0=(int)lrintf(wr[i]/s); if(v0>qmax)v0=qmax; if(v0<-8)v0=-8;
            int v1=0; if(i+1<I){ v1=(int)lrintf(wr[i+1]/s); if(v1>qmax)v1=qmax; if(v1<-8)v1=-8; }
            qr[i>>1] = (uint8_t)((v0+8) | ((v1+8)<<4));
        }
    }
}
/* quantize w[O,I] f32 -> int3-g64 (fmt=5): per 64-input group, symmetric absmax
 * (qmax=3, clamp [-4,3], stored v+4), 16B low plane + 8B high plane, ONE f32 scale
 * per group. Same math as tools/quant_ablation.py `_quant_last_dim(bits=3, group=64)`
 * (#132), here with real bit packing. */
static void pack_int3_g64(const float *w, uint8_t *q3, float *scale, int O, int I){
    int64_t ng=i3_groups(I), rb=i3_rowbytes(I);
    #pragma omp parallel for schedule(static)
    for(int o=0;o<O;o++){
        const float *wr=w+(int64_t)o*I;
        uint8_t *qr=q3+(int64_t)o*rb;
        float   *sr=scale+(int64_t)o*ng;
        for(int64_t g=0; g<ng; g++){
            int base=(int)(g*I3_GROUP), n = I-base < I3_GROUP ? I-base : I3_GROUP;
            float amax=0;
            for(int k=0;k<n;k++){ float a=fabsf(wr[base+k]); if(a>amax)amax=a; }
            float s=amax/3.f; if(s<1e-8f)s=1e-8f; sr[g]=s;
            uint8_t *lo=qr+g*I3_GBYTES, *hi=lo+16;
            memset(lo,0,I3_GBYTES);
            for(int k=0;k<n;k++){
                int v=(int)lrintf(wr[base+k]/s); if(v>3)v=3; if(v<-4)v=-4;
                unsigned u=(unsigned)(v+4);                     /* 0..7 */
                lo[k>>2] |= (uint8_t)((u&3)<<((k&3)*2));
                hi[k>>3] |= (uint8_t)(((u>>2)&1)<<(k&7));
            }
        }
    }
}

static void pack_int2(const float *w, uint8_t *q2, float *scale, int O, int I, int bits){
    int qmax=(1<<(bits-1))-1, rb=(I+3)/4;
    #pragma omp parallel for schedule(static)
    for(int o=0;o<O;o++){ const float *wr=w+(int64_t)o*I; float amax=0;
        for(int i=0;i<I;i++){ float a=fabsf(wr[i]); if(a>amax)amax=a; }
        float s=amax/qmax; if(s<1e-8f)s=1e-8f; scale[o]=s;
        uint8_t *qr=q2+(int64_t)o*rb;
        for(int i=0;i<I;i+=4){ uint8_t byte=0;
            for(int k=0;k<4 && i+k<I;k++){ int v=(int)lrintf(wr[i+k]/s); if(v>qmax)v=qmax; if(v<-2)v=-2; byte|=(uint8_t)((v+2)<<(k*2)); }
            qr[i>>2]=byte;
        }
    }
}

/* ---- fmt=6: E8/IQ3 lattice container (#452) --------------------------------
 * 98 bytes per 256 weights = 3.0625 bits/weight. Per super-block:
 *   [ 0..63]  uint8  grid index per 4-dim magnitude block
 *   [64..95]  uint32 x8 - four 7-bit sign words + 4-bit sub-scale, per 32 weights
 *   [96..97]  fp16   super-scale
 * value = d * (0.5 + code) * 0.5 * grid[idx][j] * sign, with the 8th sign of every
 * eight derived from odd parity (that is what buys the 8th bit back).
 * Byte layout and arithmetic mirror tools/iq3_pack.py exactly - that codec is the
 * oracle this kernel is tested against.
 *
 * Decode strategy: expand one 32-weight sub-block into a stack buffer, then FMA it
 * against the activations. Per-weight table lookups would dominate; per-sub-block
 * expansion keeps the grid rows (16 bytes each) hot in L1 and lets the compiler
 * vectorize the multiply-accumulate. */
#define E8_QK      256                  /* weights per super-block */
#define E8_SUB     32                   /* weights per sign/scale word */
#define E8_BBYTES  98                   /* bytes per super-block */
static inline int64_t e8_blocks(int I){ return ((int64_t)I + E8_QK - 1) / E8_QK; }
static inline int64_t e8_rowbytes(int I){ return e8_blocks(I) * E8_BBYTES; }

/* The published 256x4 magnitude grid, stored doubled (4,12,..,62 mean 2,6,..,31).
 * From ggml-common.h (MIT); tools/iq3xxs_grid.json is the same table for the
 * Python codec, and tests/test_e8_kernel.c checks the two agree through the
 * fixture. Header-local like the rest of quant.h - the engine is a single
 * translation unit and the tests include this header directly. */
static const uint8_t e8_grid[256][4] = {
    {  4,  4,  4,  4},
    { 20,  4,  4,  4},
    { 36,  4,  4,  4},
    { 12, 12,  4,  4},
    { 28, 12,  4,  4},
    { 62, 12,  4,  4},
    {  4, 20,  4,  4},
    { 20, 20,  4,  4},
    { 12, 28,  4,  4},
    { 20, 36,  4,  4},
    { 28, 62,  4,  4},
    { 44, 62,  4,  4},
    { 12,  4, 12,  4},
    { 28,  4, 12,  4},
    {  4, 12, 12,  4},
    { 20, 12, 12,  4},
    { 12, 20, 12,  4},
    { 44, 20, 12,  4},
    {  4, 28, 12,  4},
    { 20, 28, 12,  4},
    { 12, 36, 12,  4},
    { 36, 44, 12,  4},
    {  4, 62, 12,  4},
    {  4,  4, 20,  4},
    { 20,  4, 20,  4},
    { 36,  4, 20,  4},
    { 12, 12, 20,  4},
    {  4, 20, 20,  4},
    { 20, 20, 20,  4},
    { 12, 28, 20,  4},
    { 28, 28, 20,  4},
    { 62, 28, 20,  4},
    { 12, 44, 20,  4},
    { 62, 44, 20,  4},
    { 44, 62, 20,  4},
    { 12,  4, 28,  4},
    { 62,  4, 28,  4},
    {  4, 12, 28,  4},
    { 20, 12, 28,  4},
    { 44, 20, 28,  4},
    {  4, 62, 28,  4},
    { 28, 12, 36,  4},
    { 62, 28, 36,  4},
    { 36, 36, 36,  4},
    { 62, 44, 36,  4},
    { 28, 62, 36,  4},
    { 44, 62, 36,  4},
    { 12,  4, 44,  4},
    { 62,  4, 44,  4},
    { 20, 28, 44,  4},
    { 20, 44, 44,  4},
    { 44, 28, 52,  4},
    { 36, 52, 52,  4},
    {  4, 12, 62,  4},
    { 36, 12, 62,  4},
    { 52, 12, 62,  4},
    { 28, 36, 62,  4},
    { 12, 52, 62,  4},
    { 12,  4,  4, 12},
    { 28,  4,  4, 12},
    {  4, 12,  4, 12},
    { 20, 12,  4, 12},
    { 12, 20,  4, 12},
    { 28, 20,  4, 12},
    {  4, 28,  4, 12},
    { 20, 28,  4, 12},
    { 36, 28,  4, 12},
    { 62, 36,  4, 12},
    {  4, 44,  4, 12},
    {  4,  4, 12, 12},
    { 20,  4, 12, 12},
    { 12, 12, 12, 12},
    {  4, 20, 12, 12},
    { 20, 20, 12, 12},
    { 12,  4, 20, 12},
    { 28,  4, 20, 12},
    {  4, 12, 20, 12},
    { 20, 12, 20, 12},
    { 12, 20, 20, 12},
    {  4, 28, 20, 12},
    { 20, 62, 20, 12},
    {  4,  4, 28, 12},
    { 20,  4, 28, 12},
    {  4, 20, 28, 12},
    { 12, 28, 28, 12},
    { 52, 36, 28, 12},
    { 52, 52, 28, 12},
    { 12,  4, 36, 12},
    { 44,  4, 36, 12},
    {  4, 44, 36, 12},
    {  4, 20, 44, 12},
    { 36, 20, 44, 12},
    { 52, 36, 44, 12},
    { 12, 62, 44, 12},
    { 44,  4, 52, 12},
    { 20, 20, 62, 12},
    {  4, 36, 62, 12},
    {  4,  4,  4, 20},
    { 20,  4,  4, 20},
    { 12, 12,  4, 20},
    { 28, 12,  4, 20},
    {  4, 20,  4, 20},
    { 20, 20,  4, 20},
    { 52, 20,  4, 20},
    { 12, 28,  4, 20},
    { 20, 36,  4, 20},
    { 12,  4, 12, 20},
    { 28,  4, 12, 20},
    { 44,  4, 12, 20},
    {  4, 12, 12, 20},
    { 20, 12, 12, 20},
    { 12, 20, 12, 20},
    {  4, 28, 12, 20},
    { 28, 52, 12, 20},
    { 62, 52, 12, 20},
    {  4, 62, 12, 20},
    {  4,  4, 20, 20},
    { 20,  4, 20, 20},
    { 12, 12, 20, 20},
    { 62, 12, 20, 20},
    {  4, 20, 20, 20},
    { 20, 20, 20, 20},
    { 62, 28, 20, 20},
    {  4, 36, 20, 20},
    { 44, 44, 20, 20},
    { 12,  4, 28, 20},
    {  4, 12, 28, 20},
    { 36, 12, 28, 20},
    {  4, 62, 28, 20},
    { 36, 62, 28, 20},
    { 44, 28, 36, 20},
    { 28, 44, 36, 20},
    { 28,  4, 44, 20},
    { 62, 20, 44, 20},
    { 12, 36, 44, 20},
    { 36, 62, 44, 20},
    { 12,  4, 62, 20},
    { 28,  4, 62, 20},
    { 52, 12, 62, 20},
    { 44, 36, 62, 20},
    { 12,  4,  4, 28},
    {  4, 12,  4, 28},
    { 20, 12,  4, 28},
    { 12, 20,  4, 28},
    { 28, 20,  4, 28},
    {  4, 44,  4, 28},
    { 44, 52,  4, 28},
    { 20, 62,  4, 28},
    {  4,  4, 12, 28},
    { 20,  4, 12, 28},
    {  4, 20, 12, 28},
    { 12, 28, 12, 28},
    { 36, 36, 12, 28},
    { 52, 36, 12, 28},
    { 12,  4, 20, 28},
    { 28,  4, 20, 28},
    {  4, 12, 20, 28},
    { 44, 20, 20, 28},
    { 20, 44, 20, 28},
    { 20, 62, 20, 28},
    { 12, 12, 28, 28},
    { 28, 28, 28, 28},
    {  4, 28, 36, 28},
    { 62, 36, 36, 28},
    { 20, 62, 36, 28},
    {  4,  4, 44, 28},
    { 52,  4, 44, 28},
    { 20, 20, 44, 28},
    { 44, 44, 44, 28},
    { 36, 12, 52, 28},
    { 52, 28, 52, 28},
    { 28, 52, 52, 28},
    { 28, 28, 62, 28},
    {  4, 52, 62, 28},
    { 36,  4,  4, 36},
    { 62, 12,  4, 36},
    { 44, 28,  4, 36},
    { 62, 28,  4, 36},
    { 28, 44,  4, 36},
    { 62, 44,  4, 36},
    { 36, 62, 12, 36},
    {  4, 20, 20, 36},
    { 62, 28, 20, 36},
    {  4, 36, 20, 36},
    {  4, 52, 20, 36},
    { 52, 52, 20, 36},
    { 62,  4, 28, 36},
    { 44, 36, 28, 36},
    { 36,  4, 36, 36},
    { 12, 44, 36, 36},
    { 36, 52, 36, 36},
    { 44, 20, 44, 36},
    { 28, 36, 44, 36},
    {  4, 62, 44, 36},
    { 44,  4, 62, 36},
    {  4, 12, 62, 36},
    { 20, 12, 62, 36},
    {  4, 28, 62, 36},
    { 20, 12,  4, 44},
    { 12, 36,  4, 44},
    {  4, 62,  4, 44},
    {  4,  4, 12, 44},
    { 52,  4, 12, 44},
    { 52, 20, 12, 44},
    { 44, 44, 12, 44},
    { 36, 12, 20, 44},
    { 20, 28, 20, 44},
    { 20, 62, 20, 44},
    { 20,  4, 28, 44},
    { 28, 44, 28, 44},
    {  4, 12, 36, 44},
    { 28, 20, 36, 44},
    { 62, 20, 36, 44},
    { 20, 62, 36, 44},
    { 20,  4, 44, 44},
    { 12, 28, 44, 44},
    {  4, 44, 52, 44},
    { 36, 20, 62, 44},
    { 20, 36, 62, 44},
    { 36, 20,  4, 52},
    { 36, 36,  4, 52},
    { 52, 36,  4, 52},
    { 36, 52,  4, 52},
    { 12, 20, 12, 52},
    { 12, 52, 12, 52},
    { 62, 12, 20, 52},
    { 36, 52, 20, 52},
    {  4, 28, 28, 52},
    { 52, 28, 28, 52},
    { 36, 36, 36, 52},
    { 44,  4, 44, 52},
    { 20, 44, 44, 52},
    { 28, 28, 52, 52},
    { 28,  4, 62, 52},
    { 12, 20, 62, 52},
    { 28,  4,  4, 62},
    { 44,  4,  4, 62},
    { 62,  4,  4, 62},
    {  4, 12,  4, 62},
    { 20, 28,  4, 62},
    { 20, 44,  4, 62},
    { 52, 20, 12, 62},
    {  4, 36, 12, 62},
    { 20, 12, 20, 62},
    { 44, 36, 20, 62},
    { 20, 44, 20, 62},
    {  4,  4, 28, 62},
    { 44, 12, 28, 62},
    { 28, 28, 28, 62},
    {  4, 52, 28, 62},
    { 12, 20, 36, 62},
    { 12, 36, 36, 62},
    {  4,  4, 44, 62},
    { 20,  4, 44, 62},
    { 36, 20, 44, 62},
    {  4, 28, 52, 62}
};

static inline float e8_fp16_to_f32(uint16_t h){
    uint32_t sign=(uint32_t)(h>>15)<<31, exp=(h>>10)&0x1F, man=h&0x3FF, bits;
    if(!exp)      bits = man ? (sign | ((127-15+1)<<23) | (man<<13)) : sign;  /* subnormal->approx */
    else if(exp==0x1F) bits = sign | 0x7F800000u | (man<<13);
    else          bits = sign | ((exp+112)<<23) | (man<<13);
    float f; memcpy(&f,&bits,4); return f;
}

/* Expand one 32-weight sub-block. `out` must hold 32 floats. */
static inline void e8_expand_sub(const uint8_t *blk, int ib, float d, float *out){
    uint32_t word; memcpy(&word, blk + E8_QK/4 + ib*4, 4);
    float db = d * (0.5f + (float)((word>>28)&0xF)) * 0.5f;
    const uint8_t *idx = blk + ib*8;
    for(int l=0;l<4;l++){
        uint32_t seven=(word>>(7*l))&0x7F;
        const uint8_t *g0=e8_grid[idx[l*2+0]], *g1=e8_grid[idx[l*2+1]];
        int par=0;
        for(int j=0;j<8;j++){
            int neg = j<7 ? (int)((seven>>j)&1) : 0;
            if(j<7) par^=neg; else neg=par;            /* odd parity closes the block */
            float mag = (j<4 ? (float)g0[j] : (float)g1[j-4]) * 0.5f;
            out[l*8+j] = neg ? -mag*db : mag*db;
        }
    }
}

/* Fast Walsh-Hadamard transform with the per-tensor sign flip: y = Q^T x for
 * Q = D*H/sqrt(n). fmt=6 stores W@Q, so activations must be transformed before
 * the matmul (#452). Placement is the engine's job and it matters: all routed
 * experts of a layer share one input row, so ONE transform per (layer,
 * projection group) costs ~1.4 ms/token on GLM dims, while doing it per expert
 * costs ~11 ms. n must be a power of two >= the real dim; the tail is zero-pad.
 * Self-inverse up to the sign flip, so the same routine serves both directions. */
static inline void e8_fwht(float *a, int n, const uint8_t *signbits){
    if(signbits) for(int i=0;i<n;i++) if(signbits[i>>3]>>(i&7)&1) a[i]=-a[i];
    for(int len=1;len<n;len<<=1)
        for(int i=0;i<n;i+=len<<1)
            for(int j=i;j<i+len;j++){ float u=a[j],v=a[j+len]; a[j]=u+v; a[j+len]=u-v; }
    float s=1.0f/sqrtf((float)n);
    for(int i=0;i<n;i++) a[i]*=s;
}
static inline int e8_pow2_ceil(int n){ int p=1; while(p<n) p<<=1; return p; }

/* Rotation sign bits, regenerated β€” not stored. xorshift64* seeded 417+n, one
 * bit per element; tools/iq3_pack.py signs() draws the identical stream, so the
 * container carries no rotation data and the two sides cannot drift apart
 * without the oracle fixture catching it (#452). */
static inline void e8_signs(uint8_t *bits, int n){
    uint64_t s=417u+(uint64_t)n;
    for(int i=0;i<(n+7)/8;i++){
        s^=s>>12; s^=s<<25; s^=s>>27;
        bits[i]=(uint8_t)((s*2685821657736338717ULL)>>56);
    }
}
/* Apply the fmt=6 activation rotation Q^T in place, row-major [nr,dim].
 * Non-power-of-two dims tile block-diagonally; each block is the largest power
 * of two dividing the remainder (= its lowest set bit): 6144 -> 2048+4096,
 * 1536 -> 512+1024. Blocks over 32768 halve until they fit the sign buffer.
 * The converter rotates weight rows with this exact routine (W@Q and Q^T x are
 * the same transform: Q is symmetric-orthogonal up to the sign flip). */
static inline void e8_rot_rows(float *rows, int nr, int dim){
    int off=0;
    while(off<dim){
        int rem=dim-off, b=rem&(-rem);
        while(b>32768) b>>=1;
        uint8_t bits[32768/8];
        e8_signs(bits,b);
        for(int r=0;r<nr;r++) e8_fwht(rows+(int64_t)r*dim+off,b,bits);
        off+=b;
    }
}

/* ---- MXFP4 (OCP microscaling FP4) ----------------------------------------
 * The native layout of compressed-tensors "mxfp4-pack-quantized" checkpoints
 * (Kimi K3 routed experts are QAT in this format β€” pass-through, never
 * re-encoded):
 *   packed [O, I/2]  u8 β€” e2m1 nibbles, LOW nibble = even column, bit3 = sign,
 *                         bits 0..2 index {0,.5,1,1.5,2,3,4,6}
 *   scales [O, I/32] u8 β€” ue8m0 exponent per 32-column group, w = v * 2^(s-127)
 * = 4.25 bits/weight all-in. The exponent is decoded with the bit trick
 * (s<<23 as a float): exact for s in [1,254]; s=0 (2^-127, denormal) decodes
 * to +0 and s=255 to +inf on BOTH the scalar and SIMD paths, so the two stay
 * bit-identical β€” real checkpoints never contain either. */
static const float mx4_lut[16] = {0.f,.5f,1.f,1.5f,2.f,3.f,4.f,6.f,
                                  -0.f,-.5f,-1.f,-1.5f,-2.f,-3.f,-4.f,-6.f};
static inline float mx4_scale(uint8_t s){
    union { uint32_t u; float f; } b; b.u = (uint32_t)s << 23; return b.f;
}
static void matmul_mxfp4(float *y, const float *x, const uint8_t *q4, const uint8_t *e8s,
                         int S, int I, int O){
    int rb=(I+1)/2, ng=(I+31)/32;
    #pragma omp parallel for schedule(static)
    for(int o=0;o<O;o++){
        const uint8_t *w=q4+(int64_t)o*rb;
        const uint8_t *scl=e8s+(int64_t)o*ng;
        for(int s=0;s<S;s++){
            const float *xs=x+(int64_t)s*I; float a=0;
#ifdef __AVX2__
            if(I%32==0){
                /* doubled e2m1 values are exact int8 -> one pshufb decodes a
                 * nibble vector; the 0.5f un-doubling rides the group scale */
                const __m128i lut2=_mm_setr_epi8(0,1,2,3,4,6,8,12,0,-1,-2,-3,-4,-6,-8,-12);
                const __m128i m4=_mm_set1_epi8(0x0F);
                __m256 acc=_mm256_setzero_ps();
                for(int g=0;g<ng;g++){
                    __m128i by=_mm_loadu_si128((const __m128i*)(w+g*16));
                    __m128i lo=_mm_and_si128(by,m4), hi=_mm_and_si128(_mm_srli_epi16(by,4),m4);
                    __m128i n0=_mm_shuffle_epi8(lut2,_mm_unpacklo_epi8(lo,hi));  /* cols g*32+0..15  */
                    __m128i n1=_mm_shuffle_epi8(lut2,_mm_unpackhi_epi8(lo,hi));  /* cols g*32+16..31 */
                    const float *xg=xs+g*32;
                    __m256 ga=_mm256_mul_ps(_mm256_loadu_ps(xg),
                                            _mm256_cvtepi32_ps(_mm256_cvtepi8_epi32(n0)));
                    ga=_mm256_fmadd_ps(_mm256_loadu_ps(xg+8),
                                       _mm256_cvtepi32_ps(_mm256_cvtepi8_epi32(_mm_srli_si128(n0,8))),ga);
                    ga=_mm256_fmadd_ps(_mm256_loadu_ps(xg+16),
                                       _mm256_cvtepi32_ps(_mm256_cvtepi8_epi32(n1)),ga);
                    ga=_mm256_fmadd_ps(_mm256_loadu_ps(xg+24),
                                       _mm256_cvtepi32_ps(_mm256_cvtepi8_epi32(_mm_srli_si128(n1,8))),ga);
                    acc=_mm256_fmadd_ps(ga,_mm256_set1_ps(mx4_scale(scl[g])*0.5f),acc);
                }
                y[(int64_t)s*O+o]=hsum256(acc);
                continue;
            }
#endif
            for(int g=0;g<ng;g++){
                int base=g*32, glen=32; if(base+glen>I) glen=I-base;
                float sc=mx4_scale(scl[g]), ga=0;
                for(int i=base;i<base+glen;i+=2){
                    uint8_t byte=w[i>>1];
                    ga+=xs[i]*mx4_lut[byte&0xF];
                    if(i+1<base+glen) ga+=xs[i+1]*mx4_lut[byte>>4];
                }
                a+=ga*sc;
            }
            y[(int64_t)s*O+o]=a;
        }
    }
}

/* IDOT variant of matmul_mxfp4: per-32-group int8 activation quantization +
 * integer dots. The doubled e2m1 values are exact int8 (same LUT as the float
 * path), so a group reduces to maddubs(|w|, sign(x,w)) like dot_i4i8 β€” no
 * per-weight int->float conversion. Group-exact scale folding:
 *     y = sum_g idot_g * (2^(e8-127) * 0.5) * xscale_g
 * Activation-quant noise (~0.4%/group) rides on top of the e2m1 grid; gate
 * with K3_IDOT=0 in the K3 engine for exact-float A/B. */
/* Quantizzazione int8 per-gruppo-di-32 delle attivazioni per l'IDOT mxfp4.
 * Estratta da matmul_mxfp4_i8 (ordine dei loop IDENTICO -> stessi byte) così i
 * chiamanti possono sollevarla a livello layer: in kimi_k3 gate e up consumano
 * lo stesso z, e ogni expert del layer consuma le stesse righe β€” prima ogni
 * chiamata riquantizzava da capo, con un malloc/free a giro (issue del PR #1071
 * per colibri.c; qui Γ¨ la stessa malattia in forma mxfp4).
 * EN: per-32-group int8 activation quantization for the mxfp4 IDOT kernel.
 * Extracted with the loop order unchanged (bit-identical) so callers can hoist
 * it to layer level: gate/up share one z, and every expert of the layer
 * consumes the same rows. */
static void mxfp4_qx(const float *x, int S, int I, int8_t *xq, float *xsc){
    int ng=I/32;
    for(int s=0;s<S;s++)
        for(int g=0;g<ng;g++){
            const float *xg=x+(int64_t)s*I+g*32;
            float am=0; for(int i=0;i<32;i++){ float a=fabsf(xg[i]); if(a>am)am=a; }
            float sc=am/127.f; if(sc<1e-20f)sc=1e-20f;
            xsc[(int64_t)s*ng+g]=sc; float inv=1.f/sc;
            int8_t *qg=xq+(int64_t)s*I+g*32;
            for(int i=0;i<32;i++){
                int v=(int)lrintf(xg[i]*inv);
                if(v>127)v=127; if(v<-127)v=-127; qg[i]=(int8_t)v;
            }
        }
}

/* Compute su attivazioni GIA' quantizzate (xq/xsc da mxfp4_qx). Contratto: I%32==0.
 * EN: compute on pre-quantized activations. Contract: I%32==0. */
static void matmul_mxfp4_i8_pre(float *y, const int8_t *xq, const float *xsc,
                                const uint8_t *q4, const uint8_t *e8s,
                                int S, int I, int O){
    int rb=I/2, ng=I/32;
    #pragma omp parallel for schedule(static)
    for(int o=0;o<O;o++){
        const uint8_t *w=q4+(int64_t)o*rb;
        const uint8_t *scl=e8s+(int64_t)o*ng;
        for(int s=0;s<S;s++){
            const int8_t *xr=xq+(int64_t)s*I;
            const float *xsr=xsc+(int64_t)s*ng;
#ifdef __AVX2__
            {
                const __m128i lut2=_mm_setr_epi8(0,1,2,3,4,6,8,12,0,-1,-2,-3,-4,-6,-8,-12);
                const __m128i m4=_mm_set1_epi8(0x0F);
                const __m256i ones=_mm256_set1_epi16(1);
                __m256 acc=_mm256_setzero_ps();
                for(int g=0;g<ng;g++){
                    __m128i by=_mm_loadu_si128((const __m128i*)(w+g*16));
                    __m128i lo=_mm_and_si128(by,m4), hi=_mm_and_si128(_mm_srli_epi16(by,4),m4);
                    __m128i n0=_mm_shuffle_epi8(lut2,_mm_unpacklo_epi8(lo,hi));
                    __m128i n1=_mm_shuffle_epi8(lut2,_mm_unpackhi_epi8(lo,hi));
                    __m256i wv=_mm256_set_m128i(n1,n0);        /* signed doubled e2m1 */
                    __m256i xv=_mm256_loadu_si256((const __m256i*)(xr+g*32));
                    /* |w| <= 12, |x| <= 127: pair sums <= 3048, no int16 overflow */
                    __m256i p=_mm256_maddubs_epi16(_mm256_sign_epi8(wv,wv),
                                                   _mm256_sign_epi8(xv,wv));
                    acc=_mm256_fmadd_ps(_mm256_cvtepi32_ps(_mm256_madd_epi16(p,ones)),
                                        _mm256_set1_ps(mx4_scale(scl[g])*0.5f*xsr[g]),acc);
                }
                y[(int64_t)s*O+o]=hsum256(acc);
                continue;
            }
#endif
            {
                static const int8_t l2[16]={0,1,2,3,4,6,8,12,0,-1,-2,-3,-4,-6,-8,-12};
                float a=0;
                for(int g=0;g<ng;g++){
                    const int8_t *qg=xr+g*32;
                    int32_t gi=0;
                    for(int i=0;i<32;i+=2){
                        uint8_t byte=w[(g*32+i)>>1];
                        gi+=(int32_t)l2[byte&0xF]*qg[i]+(int32_t)l2[byte>>4]*qg[i+1];
                    }
                    a+=(float)gi*(mx4_scale(scl[g])*0.5f*xsr[g]);
                }
                y[(int64_t)s*O+o]=a;
            }
        }
    }
}

/* Wrapper storico: quantizza in scratch _Thread_local (niente piΓΉ malloc/free
 * per chiamata) e delega al kernel _pre. Stessi valori, stessi ordini.
 * EN: historical entry point β€” quantize into the growable thread-local scratch
 * (no more per-call malloc/free) and delegate to the _pre kernel. */
static void matmul_mxfp4_i8(float *y, const float *x, const uint8_t *q4, const uint8_t *e8s,
                            int S, int I, int O){
    if(I%32){ matmul_mxfp4(y,x,q4,e8s,S,I,O); return; }
    int ng=I/32;
    int8_t *xq; float *xsc;
    quant_scratch((size_t)S*I,(size_t)S*ng,&xq,&xsc);
    mxfp4_qx(x,S,I,xq,xsc);
    matmul_mxfp4_i8_pre(y,xq,xsc,q4,e8s,S,I,O);
}

#ifdef __ARM_NEON
/* Chemin NEON pour fmt=6 (E8/IQ3).
 *
 * Le noyau amont n'a qu'un chemin AVX2 et un repli scalaire : sur aarch64
 * (GB10 / DGX Spark, Apple silicon) tout le deballage retombe donc sur du C
 * scalaire, mesure en amont a ~92 % du temps de decodage. Cette version
 * transpose la structure du chemin AVX2, qui suit deja la forme du format :
 *
 *   - une "lane" = 8 poids = deux lignes de grille de 4 octets + 8 signes,
 *     soit exactement DEUX registres NEON de 4 flottants ;
 *   - les 8 octets de grille s'elargissent par vmovl_u8 puis vmovl_u16 ;
 *   - les 7 bits de signe stockes plus le 8e derive de la parite deviennent
 *     des masques par comparaison contre le vecteur de selection, appliques
 *     par XOR du bit de signe flottant -- aucun branchement, ce qui est
 *     precisement ce qui rendait la version scalaire couteuse ;
 *   - le 0.5 de la convention demi-unite de la grille est replie dans
 *     l'echelle du sous-bloc.
 *
 * Deux accumulateurs evitent une chaine de dependance unique sur les 32 FMA
 * d'un super-bloc, et une seule reduction horizontale par 256 poids remplace
 * une par 32.
 */
static void matmul_e8_neon(float *y, const float *x, const uint8_t *q, const float *unused,
                           int S, int I, int O){
    (void)unused;                                  /* les echelles vivent dans les blocs */
    int64_t nb=e8_blocks(I), rb=e8_rowbytes(I);
    const uint32x4_t sel_lo=(uint32x4_t){1,2,4,8};
    const uint32x4_t sel_hi=(uint32x4_t){16,32,64,128};
    const uint32x4_t sgn=vdupq_n_u32(0x80000000u);
    #pragma omp parallel for schedule(static)
    for(int o=0;o<O;o++){
        const uint8_t *wrow=q+(int64_t)o*rb;
        for(int s=0;s<S;s++){
            const float *xs=x+(int64_t)s*I;
            float acc=0;
            for(int64_t b=0;b<nb;b++){
                const uint8_t *blk=wrow+b*E8_BBYTES;
                uint16_t dh; memcpy(&dh, blk+96, 2);
                float d=e8_fp16_to_f32(dh);
                int base=(int)(b*E8_QK);
                if(base+E8_QK<=I){
                    float32x4_t ac0=vdupq_n_f32(0.0f), ac1=vdupq_n_f32(0.0f);
                    for(int ib=0; ib<E8_QK/E8_SUB; ib++){
                        uint32_t word; memcpy(&word, blk+E8_QK/4+ib*4, 4);
                        float db=d*(0.5f+(float)((word>>28)&0xF))*0.5f;
                        const float32x4_t vdb=vdupq_n_f32(0.5f*db);
                        const uint8_t *ix=blk+ib*8;
                        int off=base+ib*E8_SUB;
                        for(int l=0;l<4;l++){
                            uint32_t sv=(word>>(7*l))&0x7Fu;
                            uint32_t s8=sv|((uint32_t)__builtin_parity(sv)<<7); /* parite impaire ferme la lane */
                            uint8_t g[8];
                            memcpy(g,   e8_grid[ix[l*2+0]], 4);
                            memcpy(g+4, e8_grid[ix[l*2+1]], 4);
                            uint16x8_t w16=vmovl_u8(vld1_u8(g));
                            float32x4_t v0=vmulq_f32(vcvtq_f32_u32(vmovl_u16(vget_low_u16(w16))),  vdb);
                            float32x4_t v1=vmulq_f32(vcvtq_f32_u32(vmovl_u16(vget_high_u16(w16))), vdb);
                            uint32x4_t sd=vdupq_n_u32(s8);
                            uint32x4_t m0=vceqq_u32(vandq_u32(sd,sel_lo),sel_lo);
                            uint32x4_t m1=vceqq_u32(vandq_u32(sd,sel_hi),sel_hi);
                            v0=vreinterpretq_f32_u32(veorq_u32(vreinterpretq_u32_f32(v0),vandq_u32(m0,sgn)));
                            v1=vreinterpretq_f32_u32(veorq_u32(vreinterpretq_u32_f32(v1),vandq_u32(m1,sgn)));
                            ac0=vfmaq_f32(ac0,v0,vld1q_f32(xs+off+l*8));
                            ac1=vfmaq_f32(ac1,v1,vld1q_f32(xs+off+l*8+4));
                        }
                    }
                    acc+=vaddvq_f32(vaddq_f32(ac0,ac1));
                    continue;
                }
                /* Queue non alignee : on garde le deballage de reference. */
                for(int ib=0; ib<E8_QK/E8_SUB; ib++){
                    int off=base+ib*E8_SUB;
                    if(off>=I) break;
                    float w[E8_SUB];
                    e8_expand_sub(blk, ib, d, w);
                    int n = I-off < E8_SUB ? I-off : E8_SUB;
                    float a=0;
                    for(int k=0;k<n;k++) a += xs[off+k]*w[k];
                    acc+=a;
                }
            }
            y[(int64_t)s*O+o]=acc;
        }
    }
}
#endif /* __ARM_NEON */

static void matmul_e8(float *y, const float *x, const uint8_t *q, const float *unused,
                      int S, int I, int O){
    (void)unused;                                  /* scales live inside the blocks */
#ifdef __ARM_NEON
    /* aarch64 : le corps ci-dessous na quun chemin AVX2 et un repli scalaire,
     * donc tout retombait sur du C non vectorise. Mesure ici (expert GLM
     * 6144x2048) : 3,6x plus rapide a S=1, et 2,2x PLUS PRECIS contre une
     * reference en double precision. */
    matmul_e8_neon(y, x, q, unused, S, I, O);
    return;
#endif
    int64_t nb=e8_blocks(I), rb=e8_rowbytes(I);
    #pragma omp parallel for schedule(static)
    for(int o=0;o<O;o++){
        const uint8_t *wrow=q+(int64_t)o*rb;
        for(int s=0;s<S;s++){
            const float *xs=x+(int64_t)s*I;
            float acc=0;
            for(int64_t b=0;b<nb;b++){
                const uint8_t *blk=wrow+b*E8_BBYTES;
                uint16_t dh; memcpy(&dh, blk+96, 2);
                float d=e8_fp16_to_f32(dh);
                int base=(int)(b*E8_QK);
#ifdef __AVX2__
                /* One 8-weight lane is exactly one AVX2 register, which is what the
                 * format's own shape suggests: a lane is two 4-dim grid rows (8
                 * contiguous codebook bytes) plus 8 signs. So instead of expanding a
                 * sub-block into a stack buffer and re-reading it, each lane is
                 * decoded straight into a register and FMA'd against x:
                 *   - the 8 grid bytes widen with one vpmovzxbd,
                 *   - the sign byte (7 stored bits + the parity-derived 8th) expands
                 *     to 8 lane masks with an AND/CMPEQ against the bit-select vector
                 *     and is applied as an XOR of the float sign bit β€” no branches,
                 *     which is what made the scalar expansion expensive,
                 *   - 0.5 (the grid's half-unit convention) folds into the sub-scale.
                 * Two accumulators over the 32 FMAs of a super-block keep this off a
                 * single dependency chain, and one horizontal add per 256 weights
                 * replaces one per 32. */
                if(base+E8_QK<=I){
                    const __m256i sel=_mm256_setr_epi32(1,2,4,8,16,32,64,128);
                    const __m256i sgn=_mm256_set1_epi32((int)0x80000000u);
                    __m256 ac[2]={_mm256_setzero_ps(),_mm256_setzero_ps()};
                    for(int ib=0; ib<E8_QK/E8_SUB; ib++){
                        uint32_t word; memcpy(&word, blk+E8_QK/4+ib*4, 4);
                        float db=d*(0.5f+(float)((word>>28)&0xF))*0.5f;
                        __m256 vdb=_mm256_set1_ps(0.5f*db);
                        const uint8_t *ix=blk+ib*8;
                        int off=base+ib*E8_SUB;
                        for(int l=0;l<4;l++){
                            uint32_t sv=(word>>(7*l))&0x7Fu;
                            uint32_t s8=sv|((uint32_t)__builtin_parity(sv)<<7); /* odd parity closes the lane */
                            uint32_t g0,g1;
                            memcpy(&g0,e8_grid[ix[l*2+0]],4);
                            memcpy(&g1,e8_grid[ix[l*2+1]],4);
                            __m128i by=_mm_cvtsi64_si128((long long)((uint64_t)g0|((uint64_t)g1<<32)));
                            __m256 v=_mm256_mul_ps(_mm256_cvtepi32_ps(_mm256_cvtepu8_epi32(by)),vdb);
                            __m256i m=_mm256_cmpeq_epi32(_mm256_and_si256(_mm256_set1_epi32((int)s8),sel),sel);
                            v=_mm256_xor_ps(v,_mm256_castsi256_ps(_mm256_and_si256(m,sgn)));
                            ac[l&1]=_mm256_fmadd_ps(v,_mm256_loadu_ps(xs+off+l*8),ac[l&1]);
                        }
                    }
                    __m256 t=_mm256_add_ps(ac[0],ac[1]);
                    __m128 h=_mm_add_ps(_mm256_castps256_ps128(t),_mm256_extractf128_ps(t,1));
                    h=_mm_add_ps(h,_mm_movehl_ps(h,h));
                    h=_mm_add_ss(h,_mm_shuffle_ps(h,h,1));
                    acc+=_mm_cvtss_f32(h);
                    continue;
                }
#endif
                for(int ib=0; ib<E8_QK/E8_SUB; ib++){
                    int off=base+ib*E8_SUB;
                    if(off>=I) break;
                    float w[E8_SUB];
                    e8_expand_sub(blk, ib, d, w);
                    int n = I-off < E8_SUB ? I-off : E8_SUB;
                    float a=0;
                    for(int k=0;k<n;k++) a += xs[off+k]*w[k];
                    acc+=a;
                }
            }
            y[(int64_t)s*O+o]=acc;
        }
    }
}

#endif /* COLI_QUANT_H */