Skip to content

Run frontier MoE models on hardware you already own β€” pure C, zero deps, experts streamed from disk. Tiny engine, immense model. 🐦

C

perf(quant): multi-row tile + AVX-512 + AMX arms for the K1b grouped IDOT

Judeb4292dd08963
diff --git a/c/Makefile b/c/Makefile
index 6cfa5cf7..941a8038 100644
--- a/c/Makefile
+++ b/c/Makefile
@@ -1892,6 +1892,11 @@ tests/bench_indexer_allocations$(EXE): tests/bench_indexer_allocations.c
tests/bench_idot$(EXE): tests/bench_idot.c colibri.c st.h uring.h json.h tok.h tok_unicode.h compat.h grammar.h tier.h quant.h sample.h kv_persist.h telemetry.h route_trace.h
$(CC) $(CFLAGS) $< -o $@ $(LDFLAGS)
+# bench_i4p_gidot: microbenchmark (per-row vs multi-row/AMX K1b grouped planar IDOT), NOT a test gate.
+# Build on demand: make tests/bench_i4p_gidot ARCH=native
+tests/bench_i4p_gidot$(EXE): tests/bench_i4p_gidot.c colibri.c st.h uring.h json.h tok.h tok_unicode.h compat.h grammar.h tier.h quant.h sample.h kv_persist.h telemetry.h route_trace.h
+ $(CC) $(CFLAGS) $< -o $@ $(LDFLAGS)
+
# bench_gemv_stream: microbenchmark (decode-regime GEMV bandwidth vs the read ceiling;
# frozen-baseline + deinterleaved-x candidate A/B), NOT a test gate.
# Build on demand: make tests/bench_gemv_stream ARCH=native
diff --git a/c/colibri.c b/c/colibri.c
index 24ab3b3a..11942714 100644
--- a/c/colibri.c
+++ b/c/colibri.c
@@ -1066,17 +1066,22 @@ static void matmul_i4_grouped_pair(float *yg, float *yu, const float *x,
* i loro consumatori sono esattamente matmul_qt/expert_gate_up, tutti
* planar-aware. kv_b (letto raw dal path DSA fuso), embedding (dequant
* per-token) e attenzione restano a coppie PER COSTRUZIONE. Off su build GPU
- * (i backend leggono q4 a coppie), sotto XEXP (dot_i4i8 sulle slab) e su
- * build AVX-512F (il ramo f32 a 512 bit accumula in altro ordine).
- * EN: K1 planar gate. MoE tensors only; GPU builds, XEXP and AVX-512F builds
- * keep the classic pair layout. PLANAR=0 is the kill switch. */
+ * (i backend leggono q4 a coppie) e sotto XEXP (dot_i4i8 sulle slab). Su
+ * build AVX-512F il ramo f32 a 512 bit accumula in altro ordine, quindi il
+ * planare fmt=2 resta SPENTO (vedi qt_planarize); la famiglia K1b (fmt=4,
+ * somme intere, esatte a qualunque larghezza) invece si accende β€” e' il path
+ * IDOT dei checkpoint gs64 sui server AVX-512/AMX.
+ * EN: K1 planar gate. MoE tensors only; GPU builds and XEXP keep the classic
+ * pair layout. On AVX-512F builds only fmt=2 planarization stays off (the
+ * 512-bit f32 pair arm accumulates in a different order than the planar f32
+ * twin); the integer K1b family (fmt=4) is width-exact and now enabled there,
+ * so IDOT_GS=1 reaches gs64 checkpoints on AVX-512/AMX servers.
+ * PLANAR=0 is the kill switch. */
static int g_planar=-1;
static int planar_on(void){
if(g_planar<0){
#if defined(COLI_CUDA)||defined(COLI_METAL)||defined(COLI_VULKAN)
g_planar=0;
-#elif defined(__AVX512F__)&&defined(__AVX512BW__)
- g_planar=0;
#elif !defined(__AVX2__)
g_planar=0; /* matmul_i4p non ha (ancora) un ramo NEON: su ARM il path
* f32 planare degraderebbe a scalare E cambierebbe l'ordine
@@ -1109,6 +1114,14 @@ static void qt_planarize(QT *t){
planarize_i4(t->q4,t->O,t->I); t->planar=1; return;
}
if(t->fmt!=2) return;
+#if defined(__AVX512F__)&&defined(__AVX512BW__)
+ /* fmt=2 stays a coppie qui: il gemello f32 planare replica l'ordine di
+ * accumulo AVX2, non quello del ramo dot_i4f_avx512 a 512 bit β€” il claim
+ * bit-identico della famiglia f32 vale solo dove i gemelli coincidono.
+ * EN: fmt=2 keeps the pair layout on AVX-512 builds; the f32 planar twin
+ * mirrors the AVX2 accumulation order, not the 512-bit f32 arm's. */
+ return;
+#endif
planarize_i4(t->q4,t->O,t->I); t->planar=1;
if(atomic_fetch_add_explicit(&g_planar_n,1,memory_order_relaxed)==0)
fprintf(stderr,"[K1] planar int4 layout active (PLANAR=0 disables)\n");
diff --git a/c/quant.h b/c/quant.h
index f53ad436..a011ea93 100644
--- a/c/quant.h
+++ b/c/quant.h
@@ -1105,33 +1105,290 @@ static inline int32_t dot_i4p_u(const uint8_t *w4, const int8_t *x, int I){
* EN: grouped planar IDOT, opt-in. With gs=64 the scale group IS the plane
* block; per-group unsigned dot minus 8*group-sum, times the group scale.
* int8 activations: not bit-identical to the f32 grouped kernel, hence the
- * flag until the ablation blesses a default. */
-static void matmul_i4p_grouped_idot(float *y, const int8_t *xq, const float *sx,
- const int32_t *xsg, const uint8_t *q4,
- const float *scale, int S, int I, int O, int gs){
+ * flag until the ablation blesses a default.
+ *
+ * Three execution shapes below, one contract: per (row, output) the float
+ * accumulation is the SAME sequence of fmaf((float)group_int, scale[g], a)
+ * in ascending g, and every group_int is an exact int32 β€” so the per-row
+ * path, the 1x4 row tile, and the AMX tile are bit-identical to each other
+ * and to the pure-C reference on every ISA. */
+
+/* one planar 64-element block, unpacked once and shared across a row tile:
+ * lo nibbles = elements base..base+31 in order, hi = base+32..base+63. */
+#if defined(coli_dpbusd256)
+static inline void i4p_blk256(const uint8_t *blk, __m256i *lo, __m256i *hi){
+ const __m256i m4=_mm256_set1_epi8(0x0F);
+ __m256i b=_mm256_loadu_si256((const __m256i*)blk);
+ *lo=_mm256_and_si256(b,m4);
+ *hi=_mm256_and_si256(_mm256_srli_epi16(b,4),m4);
+}
+#endif
+#if defined(__AVX512VNNI__) && defined(__AVX512BW__)
+/* whole block in one zmm: lanes 0..31 = lo elements, 32..63 = hi β€” matches a
+ * straight 64-byte load of the activation block, so ONE dpbusd per block. */
+static inline __m512i i4p_blk512(const uint8_t *blk){
+ const __m256i m4=_mm256_set1_epi8(0x0F);
+ __m256i b=_mm256_loadu_si256((const __m256i*)blk);
+ return _mm512_inserti64x4(_mm512_castsi256_si512(_mm256_and_si256(b,m4)),
+ _mm256_and_si256(_mm256_srli_epi16(b,4),m4),1);
+}
+#endif
+
+/* ---- K1c: AMX int8 tile kernel (Sapphire Rapids+, opt-in via the same
+ * IDOT_GS=1 family gate; AMX=0 kills it, AMX_S_MIN sets the row threshold).
+ * With gs a multiple of 64, K=64 tile-multiplies cover a scale group exactly:
+ * B tile = 16 output rows' int4 block unpacked once to SIGNED int8 (v-8, so
+ * tdpbssd returns d - 8*sum(x_group) directly β€” the same int32 the vector
+ * path computes as d_unsigned - 8*xsg), A tile = up to 16 activation rows.
+ * The unpack cost is paid once per (output tile, group) and amortized over
+ * every activation row β€” the multi-row reuse the pair-layout kernels lack.
+ * Linux-only arming: tile data needs an ARCH_REQ_XCOMP_PERM handshake. */
+#if defined(__AMX_INT8__) && defined(__AMX_TILE__) && defined(__AVX512F__)
+#define COLI_HAVE_AMX_I4P 1
+#if defined(__linux__)
+#include <unistd.h>
+#include <sys/syscall.h>
+#elif defined(_WIN32)
+#ifndef WIN32_LEAN_AND_MEAN
+#define WIN32_LEAN_AND_MEAN
+#endif
+#include <windows.h>
+#endif
+static int coli_amx_state=-1;
+static int coli_amx_s_min=8;
+static int coli_amx_ok(void){
+ if(coli_amx_state<0){
+ const char *e=getenv("AMX");
+ const char *sm=getenv("AMX_S_MIN"); if(sm&&atoi(sm)>0) coli_amx_s_min=atoi(sm);
+#if defined(__linux__)
+ /* ARCH_REQ_XCOMP_PERM(XFEATURE_XTILEDATA): kernel >= 5.16 grants tile
+ * state per-process; without it the first tile op SIGILLs. */
+ coli_amx_state=(e&&*e=='0')?0:(syscall(SYS_arch_prctl,0x1023,18)==0);
+#elif defined(_WIN32)
+ /* Windows 11: tile data is an opt-in per-process XState feature.
+ * Dynamic lookup keeps older kernels building/running (arming just
+ * fails closed there). XSTATE_AMX_TILE_DATA = 18. */
+ if(e&&*e=='0') coli_amx_state=0;
+ else {
+ HMODULE k32=GetModuleHandleA("kernel32.dll");
+ typedef BOOL (WINAPI *coli_xstate_fn)(ULONG64);
+ coli_xstate_fn f=k32?(coli_xstate_fn)(void*)GetProcAddress(k32,"EnableProcessOptionalXStateFeatures"):NULL;
+ coli_amx_state=(f && f(1ull<<18))?1:0;
+ }
+#else
+ (void)e; coli_amx_state=0;
+#endif
+ if(coli_amx_state)
+ fprintf(stderr,"[K1c] AMX int8 tile kernel armed for gs64 tensors "
+ "(AMX=0 disables, engages at S>=%d)\n",coli_amx_s_min);
+ }
+ return coli_amx_state;
+}
+/* ldtilecfg layout (palette 1): tmm0=C [rows x 16 i32], tmm1=A [rows x 64 i8],
+ * tmm2=B [16 x 64 i8 VNNI]. rows<16 reconfigures for the last partial s-tile. */
+struct coli_tilecfg { uint8_t palette,start_row,rsvd[14]; uint16_t colsb[16]; uint8_t rows[16]; };
+static void coli_amx_cfg(int arows){
+ struct coli_tilecfg c; memset(&c,0,sizeof c); c.palette=1;
+ c.rows[0]=(uint8_t)arows; c.colsb[0]=64;
+ c.rows[1]=(uint8_t)arows; c.colsb[1]=64;
+ c.rows[2]=16; c.colsb[2]=64;
+ _tile_loadconfig(&c);
+}
+static void matmul_i4p_gidot_amx(float *y, const int8_t *xq, const float *sx,
+ const uint8_t *q4, const float *scale,
+ int S, int I, int O16, int O, int gs){
+ int rb=(I+1)/2, ng=(I+gs-1)/gs, bpg=gs/64;
+ #pragma omp parallel
+ {
+ float *acc=malloc((size_t)S*16*sizeof(float));
+ if(!acc){ fprintf(stderr,"OOM: amx acc\n"); exit(1); }
+ int8_t bstage[4*1024] __attribute__((aligned(64))); /* bpg<=4 gated below */
+ int32_t cbuf[16*16] __attribute__((aligned(64)));
+ float sclT[16];
+ int cur=16; coli_amx_cfg(16);
+ #pragma omp for schedule(static)
+ for(int ot=0; ot<O16; ot+=16){
+ memset(acc,0,(size_t)S*16*sizeof(float));
+ for(int g=0; g<ng; g++){
+ for(int j=0;j<16;j++) sclT[j]=scale[(int64_t)(ot+j)*ng+g];
+ for(int b=0;b<bpg;b++){
+ /* B tile, VNNI [k/4][n][4]: element k of output row ot+n
+ * lands at row k/4, byte n*4+k%4 β€” signed (v-8) at unpack. */
+ int8_t *dst=bstage+(size_t)b*1024;
+ int64_t boff=((int64_t)g*gs+b*64)>>1;
+ for(int n=0;n<16;n++){
+ const uint8_t *blk=q4+(int64_t)(ot+n)*rb+boff;
+ for(int k=0;k<32;k++){
+ dst[(k>>2)*64+n*4+(k&3)] =(int8_t)((blk[k]&0xF)-8);
+ dst[((k+32)>>2)*64+n*4+((k+32)&3)]=(int8_t)((blk[k]>>4)-8);
+ }
+ }
+ }
+ for(int st=0; st<S; st+=16){
+ int rows=S-st<16?S-st:16;
+ if(rows!=cur){ coli_amx_cfg(rows); cur=rows; }
+ _tile_zero(0);
+ for(int b=0;b<bpg;b++){
+ _tile_loadd(1, xq+(int64_t)st*I+(int64_t)g*gs+b*64, (size_t)I);
+ _tile_loadd(2, bstage+(size_t)b*1024, 64);
+ _tile_dpbssd(0,1,2);
+ }
+ _tile_stored(0, cbuf, 64);
+ /* per-lane fmadd == the vector path's per-(s,o) scalar fmaf:
+ * same values, same ascending-g order, single rounding. */
+ for(int r=0;r<rows;r++){
+ __m512 cv=_mm512_cvtepi32_ps(_mm512_load_si512((const void*)(cbuf+(size_t)r*16)));
+ __m512 av=_mm512_loadu_ps(acc+(int64_t)(st+r)*16);
+ _mm512_storeu_ps(acc+(int64_t)(st+r)*16,
+ _mm512_fmadd_ps(cv,_mm512_loadu_ps(sclT),av));
+ }
+ }
+ }
+ for(int s=0;s<S;s++)
+ _mm512_storeu_ps(y+(int64_t)s*O+ot,
+ _mm512_mul_ps(_mm512_loadu_ps(acc+(int64_t)s*16),
+ _mm512_set1_ps(sx[s])));
+ }
+ _tile_release();
+ free(acc);
+ }
+}
+#endif
+
+/* vector body over an output range [o0,o1): the 1x4 row tile pays the weight
+ * block's load+mask once per 4 activation rows (the fmt=2 K2 tile's idea,
+ * brought to the grouped family β€” the prefill batch-union and the serve mux
+ * deliver exactly these multi-row calls). Integer group dots in any lane
+ * order are exact, and each row keeps its own ascending-g fmaf chain. */
+static void i4p_gidot_rows(float *y, const int8_t *xq, const float *sx,
+ const int32_t *xsg, const uint8_t *q4,
+ const float *scale, int S, int I, int O, int gs,
+ int o0, int o1){
int rb=(I+1)/2, ng=(I+gs-1)/gs, bpg=gs/64; /* blocchi-piano per gruppo */
#pragma omp parallel for schedule(static)
- for(int o=0;o<O;o++){
+ for(int o=o0;o<o1;o++){
const uint8_t *w=q4+(int64_t)o*rb;
const float *scl=scale+(int64_t)o*ng;
- for(int s=0;s<S;s++){
+ int s=0;
+ for(; s+4<=S; s+=4){
+ const int8_t *x0=xq +(int64_t)s*I, *x1=x0+I, *x2=x1+I, *x3=x2+I;
+ const int32_t *g0=xsg+(int64_t)s*ng, *g1=g0+ng, *g2=g1+ng, *g3=g2+ng;
+ float a0=0,a1=0,a2=0,a3=0;
+ int g=0;
+ for(; (g+1)*gs<=I; g++){
+ int32_t d0,d1,d2,d3;
+#if defined(__AVX512VNNI__) && defined(__AVX512BW__)
+ __m512i A0=_mm512_setzero_si512(),A1=_mm512_setzero_si512();
+ __m512i A2=_mm512_setzero_si512(),A3=_mm512_setzero_si512();
+ for(int b=0;b<bpg;b++){
+ int base=g*gs+b*64;
+ __m512i wz=i4p_blk512(w+(base>>1));
+ A0=_mm512_dpbusd_epi32(A0,wz,_mm512_loadu_si512((const void*)(x0+base)));
+ A1=_mm512_dpbusd_epi32(A1,wz,_mm512_loadu_si512((const void*)(x1+base)));
+ A2=_mm512_dpbusd_epi32(A2,wz,_mm512_loadu_si512((const void*)(x2+base)));
+ A3=_mm512_dpbusd_epi32(A3,wz,_mm512_loadu_si512((const void*)(x3+base)));
+ }
+ d0=_mm512_reduce_add_epi32(A0); d1=_mm512_reduce_add_epi32(A1);
+ d2=_mm512_reduce_add_epi32(A2); d3=_mm512_reduce_add_epi32(A3);
+#elif defined(coli_dpbusd256)
+ __m256i A0=_mm256_setzero_si256(),A1=_mm256_setzero_si256();
+ __m256i A2=_mm256_setzero_si256(),A3=_mm256_setzero_si256();
+ for(int b=0;b<bpg;b++){
+ int base=g*gs+b*64; __m256i lo,hi;
+ i4p_blk256(w+(base>>1),&lo,&hi);
+ A0=coli_dpbusd256(A0,lo,_mm256_loadu_si256((const __m256i*)(x0+base)));
+ A0=coli_dpbusd256(A0,hi,_mm256_loadu_si256((const __m256i*)(x0+base+32)));
+ A1=coli_dpbusd256(A1,lo,_mm256_loadu_si256((const __m256i*)(x1+base)));
+ A1=coli_dpbusd256(A1,hi,_mm256_loadu_si256((const __m256i*)(x1+base+32)));
+ A2=coli_dpbusd256(A2,lo,_mm256_loadu_si256((const __m256i*)(x2+base)));
+ A2=coli_dpbusd256(A2,hi,_mm256_loadu_si256((const __m256i*)(x2+base+32)));
+ A3=coli_dpbusd256(A3,lo,_mm256_loadu_si256((const __m256i*)(x3+base)));
+ A3=coli_dpbusd256(A3,hi,_mm256_loadu_si256((const __m256i*)(x3+base+32)));
+ }
+ d0=hsum256_i32(A0); d1=hsum256_i32(A1);
+ d2=hsum256_i32(A2); d3=hsum256_i32(A3);
+#elif defined(__AVX2__)
+ const __m256i ones=_mm256_set1_epi16(1);
+ const __m256i m4=_mm256_set1_epi8(0x0F);
+ __m256i A0=_mm256_setzero_si256(),A1=_mm256_setzero_si256();
+ __m256i A2=_mm256_setzero_si256(),A3=_mm256_setzero_si256();
+ for(int b=0;b<bpg;b++){
+ int base=g*gs+b*64;
+ __m256i bb=_mm256_loadu_si256((const __m256i*)(w+(base>>1)));
+ __m256i lo=_mm256_and_si256(bb,m4);
+ __m256i hi=_mm256_and_si256(_mm256_srli_epi16(bb,4),m4);
+ /* maddubs(u8,s8): u<=15, |x|<=127 -> pair <= 3810, int16-safe */
+ A0=_mm256_add_epi32(A0,_mm256_madd_epi16(_mm256_maddubs_epi16(lo,_mm256_loadu_si256((const __m256i*)(x0+base))),ones));
+ A0=_mm256_add_epi32(A0,_mm256_madd_epi16(_mm256_maddubs_epi16(hi,_mm256_loadu_si256((const __m256i*)(x0+base+32))),ones));
+ A1=_mm256_add_epi32(A1,_mm256_madd_epi16(_mm256_maddubs_epi16(lo,_mm256_loadu_si256((const __m256i*)(x1+base))),ones));
+ A1=_mm256_add_epi32(A1,_mm256_madd_epi16(_mm256_maddubs_epi16(hi,_mm256_loadu_si256((const __m256i*)(x1+base+32))),ones));
+ A2=_mm256_add_epi32(A2,_mm256_madd_epi16(_mm256_maddubs_epi16(lo,_mm256_loadu_si256((const __m256i*)(x2+base))),ones));
+ A2=_mm256_add_epi32(A2,_mm256_madd_epi16(_mm256_maddubs_epi16(hi,_mm256_loadu_si256((const __m256i*)(x2+base+32))),ones));
+ A3=_mm256_add_epi32(A3,_mm256_madd_epi16(_mm256_maddubs_epi16(lo,_mm256_loadu_si256((const __m256i*)(x3+base))),ones));
+ A3=_mm256_add_epi32(A3,_mm256_madd_epi16(_mm256_maddubs_epi16(hi,_mm256_loadu_si256((const __m256i*)(x3+base+32))),ones));
+ }
+ d0=hsum256_i32(A0); d1=hsum256_i32(A1);
+ d2=hsum256_i32(A2); d3=hsum256_i32(A3);
+#else
+ d0=d1=d2=d3=0;
+ for(int b=0;b<bpg;b++){
+ int base=g*gs+b*64;
+ const uint8_t *blk=w+(base>>1);
+ for(int k=0;k<32;k++){
+ int32_t ul=(int32_t)(blk[k]&0xF), uh=(int32_t)(blk[k]>>4);
+ d0+=ul*x0[base+k]+uh*x0[base+k+32];
+ d1+=ul*x1[base+k]+uh*x1[base+k+32];
+ d2+=ul*x2[base+k]+uh*x2[base+k+32];
+ d3+=ul*x3[base+k]+uh*x3[base+k+32];
+ }
+ }
+#endif
+ a0=fmaf((float)(d0-8*g0[g]),scl[g],a0);
+ a1=fmaf((float)(d1-8*g1[g]),scl[g],a1);
+ a2=fmaf((float)(d2-8*g2[g]),scl[g],a2);
+ a3=fmaf((float)(d3-8*g3[g]),scl[g],a3);
+ }
+ if(g*gs<I){ /* coda: gruppo parziale, nibble a coppie */
+ int32_t d0=0,d1=0,d2=0,d3=0;
+ for(int i=g*gs;i<I;i++){
+ uint8_t byte=w[i>>1];
+ int32_t u=(int32_t)((i&1)?(byte>>4):(byte&0xF));
+ d0+=u*x0[i]; d1+=u*x1[i]; d2+=u*x2[i]; d3+=u*x3[i];
+ }
+ a0=fmaf((float)(d0-8*g0[g]),scl[g],a0);
+ a1=fmaf((float)(d1-8*g1[g]),scl[g],a1);
+ a2=fmaf((float)(d2-8*g2[g]),scl[g],a2);
+ a3=fmaf((float)(d3-8*g3[g]),scl[g],a3);
+ }
+ y[(int64_t)s*O+o] =a0*sx[s];
+ y[(int64_t)(s+1)*O+o]=a1*sx[s+1];
+ y[(int64_t)(s+2)*O+o]=a2*sx[s+2];
+ y[(int64_t)(s+3)*O+o]=a3*sx[s+3];
+ }
+ for(; s<S; s++){
const int8_t *xr=xq+(int64_t)s*I;
const int32_t *xg=xsg+(int64_t)s*ng;
float a=0; int g=0;
for(; (g+1)*gs<=I; g++){ /* gruppi interi */
int32_t d=0;
+#if defined(__AVX512VNNI__) && defined(__AVX512BW__)
+ __m512i acc=_mm512_setzero_si512();
+ for(int b=0;b<bpg;b++){
+ int base=g*gs+b*64;
+ acc=_mm512_dpbusd_epi32(acc,i4p_blk512(w+(base>>1)),
+ _mm512_loadu_si512((const void*)(xr+base)));
+ }
+ d=_mm512_reduce_add_epi32(acc);
+#else
for(int b=0;b<bpg;b++){
int base=g*gs+b*64;
const uint8_t *blk=w+(base>>1);
const int8_t *xb=xr+base;
#if defined(coli_dpbusd256)
- const __m256i m4=_mm256_set1_epi8(0x0F);
- __m256i bb=_mm256_loadu_si256((const __m256i*)blk);
+ __m256i lo,hi; i4p_blk256(blk,&lo,&hi);
__m256i acc=_mm256_setzero_si256();
- acc=coli_dpbusd256(acc,_mm256_and_si256(bb,m4),
- _mm256_loadu_si256((const __m256i*)xb));
- acc=coli_dpbusd256(acc,_mm256_and_si256(_mm256_srli_epi16(bb,4),m4),
- _mm256_loadu_si256((const __m256i*)(xb+32)));
+ acc=coli_dpbusd256(acc,lo,_mm256_loadu_si256((const __m256i*)xb));
+ acc=coli_dpbusd256(acc,hi,_mm256_loadu_si256((const __m256i*)(xb+32)));
d+=hsum256_i32(acc);
#elif defined(__AVX2__)
const __m256i m4=_mm256_set1_epi8(0x0F);
@@ -1151,6 +1408,7 @@ static void matmul_i4p_grouped_idot(float *y, const int8_t *xq, const float *sx,
}
#endif
}
+#endif
a=fmaf((float)(d-8*xg[g]),scl[g],a);
}
if(g*gs<I){ /* coda: gruppo parziale, nibble a coppie */
@@ -1166,6 +1424,24 @@ static void matmul_i4p_grouped_idot(float *y, const int8_t *xq, const float *sx,
}
}
+static void matmul_i4p_grouped_idot(float *y, const int8_t *xq, const float *sx,
+ const int32_t *xsg, const uint8_t *q4,
+ const float *scale, int S, int I, int O, int gs){
+#if defined(COLI_HAVE_AMX_I4P)
+ /* AMX takes the aligned bulk (full 16-output tiles, full groups only β€”
+ * the real gs64 checkpoints have I%gs==0); the vector path finishes any
+ * output remainder. Below the row threshold the B-tile unpack does not
+ * amortize and the vector tile is the better kernel. */
+ if(coli_amx_ok() && S>=coli_amx_s_min && gs%64==0 && gs<=256 && I%gs==0 && O>=16){
+ int O16=O&~15;
+ matmul_i4p_gidot_amx(y,xq,sx,q4,scale,S,I,O16,O,gs);
+ if(O16<O) i4p_gidot_rows(y,xq,sx,xsg,q4,scale,S,I,O,gs,O16,O);
+ return;
+ }
+#endif
+ i4p_gidot_rows(y,xq,sx,xsg,q4,scale,S,I,O,gs,0,O);
+}
+
/* matmul IDOT planare (fmt=2): y = (dot_u - 8*xsum[s]) * scale[o] * sx[s].
* Bit-identico a matmul_i4_idot: somme intere, identita' esatta. */
static void matmul_i4p_idot(float *y, const int8_t *xq, const float *sx, const int32_t *xsum,
diff --git a/c/tests/bench_i4p_gidot.c b/c/tests/bench_i4p_gidot.c
new file mode 100644
index 00000000..4843a070
--- /dev/null
+++ b/c/tests/bench_i4p_gidot.c
@@ -0,0 +1,135 @@
+/* Microbenchmark: per-row vs multi-row K1b grouped planar IDOT (quant.h).
+ * NOT a unit test -- test_int_kernel_exact.c proves correctness/bit-equality.
+ *
+ * This measures the multi-row claim behind the fmt=4 tile work: the per-row
+ * kernel re-loads and re-masks every weight block once per activation row, so
+ * batched calls (prefill batch-union rows, the serve mux's decode batch) pay
+ * S times the weight traffic. The 1x4 tile (and AMX on Sapphire Rapids) pays
+ * it once per tile. The OLD kernel below is a verbatim copy of the pre-tile
+ * body; the NEW one is the real dispatcher, so on an AMX host this also
+ * benches the tile-unit path (AMX_S_MIN gates it; force with AMX_S_MIN=2).
+ *
+ * Run: make tests/bench_i4p_gidot ARCH=native && ./tests/bench_i4p_gidot
+ * (not in TEST_BINS -- not a gate) */
+#define main coli_glm_main_unused
+#include "../colibri.c"
+#undef main
+#include <stdint.h>
+#include <string.h>
+
+static uint32_t rs=0x2545F491u;
+static uint32_t xr(void){ rs^=rs<<13; rs^=rs>>17; rs^=rs<<5; return rs; }
+
+/* ---- OLD kernel: verbatim copy of the pre-tile per-row body ---- */
+static void gidot_old(float *y, const int8_t *xq, const float *sx,
+ const int32_t *xsg, const uint8_t *q4,
+ const float *scale, int S, int I, int O, int gs){
+ int rb=(I+1)/2, ng=(I+gs-1)/gs, bpg=gs/64;
+ #pragma omp parallel for schedule(static)
+ for(int o=0;o<O;o++){
+ const uint8_t *w=q4+(int64_t)o*rb;
+ const float *scl=scale+(int64_t)o*ng;
+ for(int s=0;s<S;s++){
+ const int8_t *xr2=xq+(int64_t)s*I;
+ const int32_t *xg=xsg+(int64_t)s*ng;
+ float a=0; int g=0;
+ for(; (g+1)*gs<=I; g++){
+ int32_t d=0;
+ for(int b=0;b<bpg;b++){
+ int base=g*gs+b*64;
+ const uint8_t *blk=w+(base>>1);
+ const int8_t *xb=xr2+base;
+#if defined(coli_dpbusd256)
+ const __m256i m4=_mm256_set1_epi8(0x0F);
+ __m256i bb=_mm256_loadu_si256((const __m256i*)blk);
+ __m256i acc=_mm256_setzero_si256();
+ acc=coli_dpbusd256(acc,_mm256_and_si256(bb,m4),
+ _mm256_loadu_si256((const __m256i*)xb));
+ acc=coli_dpbusd256(acc,_mm256_and_si256(_mm256_srli_epi16(bb,4),m4),
+ _mm256_loadu_si256((const __m256i*)(xb+32)));
+ d+=hsum256_i32(acc);
+#elif defined(__AVX2__)
+ const __m256i m4=_mm256_set1_epi8(0x0F);
+ const __m256i ones=_mm256_set1_epi16(1);
+ __m256i bb=_mm256_loadu_si256((const __m256i*)blk);
+ __m256i p0=_mm256_maddubs_epi16(_mm256_and_si256(bb,m4),
+ _mm256_loadu_si256((const __m256i*)xb));
+ __m256i p1=_mm256_maddubs_epi16(_mm256_and_si256(_mm256_srli_epi16(bb,4),m4),
+ _mm256_loadu_si256((const __m256i*)(xb+32)));
+ __m256i acc=_mm256_add_epi32(_mm256_madd_epi16(p0,ones),
+ _mm256_madd_epi16(p1,ones));
+ d+=hsum256_i32(acc);
+#else
+ for(int k=0;k<32;k++){
+ d+=(int32_t)(blk[k]&0xF)*xb[k];
+ d+=(int32_t)(blk[k]>>4)*xb[k+32];
+ }
+#endif
+ }
+ a=fmaf((float)(d-8*xg[g]),scl[g],a);
+ }
+ if(g*gs<I){
+ int32_t d=0;
+ for(int i=g*gs;i<I;i++){
+ uint8_t byte=w[i>>1];
+ d+=(int32_t)((i&1)?(byte>>4):(byte&0xF))*xr2[i];
+ }
+ a=fmaf((float)(d-8*xg[g]),scl[g],a);
+ }
+ y[(int64_t)s*O+o]=a*sx[s];
+ }
+ }
+}
+
+#define O_DIM 2048
+#define I_DIM 2048
+#define GS 64
+#define REPS 400
+static int cmp_d(const void*a,const void*b){ double x=*(const double*)a,y=*(const double*)b; return x<y?-1:x>y?1:0; }
+
+int main(void){
+ int rb=(I_DIM+1)/2, ng=(I_DIM+GS-1)/GS;
+ int Ss[]={1,4,8,16};
+ uint8_t *q4=malloc((size_t)O_DIM*rb);
+ float *sc=malloc((size_t)O_DIM*ng*sizeof(float));
+ for(size_t i=0;i<(size_t)O_DIM*rb;i++) q4[i]=(uint8_t)(xr()&0xFF);
+ planarize_i4(q4,O_DIM,I_DIM);
+ for(size_t i=0;i<(size_t)O_DIM*ng;i++) sc[i]=0.0005f+(xr()%911)*1e-6f;
+
+ int Smax=16;
+ int8_t *xq=malloc((size_t)Smax*I_DIM);
+ float *sx=malloc(Smax*sizeof(float));
+ int32_t *xsg=malloc((size_t)Smax*ng*sizeof(int32_t));
+ for(size_t i=0;i<(size_t)Smax*I_DIM;i++){ int v=(int)(xr()%255)-127; xq[i]=(int8_t)v; }
+ for(int s=0;s<Smax;s++){ sx[s]=0.01f+(xr()%89)*1e-4f;
+ for(int g=0;g<ng;g++){ int32_t a=0;
+ for(int i=g*GS;i<(g+1)*GS;i++) a+=xq[(size_t)s*I_DIM+i];
+ xsg[(size_t)s*ng+g]=a; } }
+ float *ya=malloc((size_t)Smax*O_DIM*sizeof(float));
+ float *yb=malloc((size_t)Smax*O_DIM*sizeof(float));
+ static double t[REPS];
+
+ printf("bench_i4p_gidot: O=%d I=%d gs=%d, %s, weights %.1f MB\n",
+ O_DIM,I_DIM,GS,IDOT_KERNEL,(double)O_DIM*rb/1e6);
+ for(size_t k=0;k<sizeof(Ss)/sizeof(*Ss);k++){
+ int S=Ss[k];
+ gidot_old(ya,xq,sx,xsg,q4,sc,S,I_DIM,O_DIM,GS);
+ matmul_i4p_grouped_idot(yb,xq,sx,xsg,q4,sc,S,I_DIM,O_DIM,GS);
+ if(memcmp(ya,yb,(size_t)S*O_DIM*sizeof(float))){ fprintf(stderr,"MISMATCH S=%d\n",S); return 1; }
+ double med[2];
+ for(int which=0;which<2;which++){
+ for(int wu=0;wu<20;wu++) (which?matmul_i4p_grouped_idot:gidot_old)(yb,xq,sx,xsg,q4,sc,S,I_DIM,O_DIM,GS);
+ for(int r=0;r<REPS;r++){
+ double t0=now_s();
+ (which?matmul_i4p_grouped_idot:gidot_old)(yb,xq,sx,xsg,q4,sc,S,I_DIM,O_DIM,GS);
+ t[r]=now_s()-t0;
+ }
+ qsort(t,REPS,sizeof(double),cmp_d);
+ med[which]=t[REPS/2];
+ }
+ double wb=(double)O_DIM*rb; /* weight bytes touched once per call */
+ printf("S=%2d old %8.1f us (%6.2f GB/s-w) new %8.1f us (%6.2f GB/s-w) new/old %.2fx\n",
+ S, med[0]*1e6, wb/med[0]/1e9, med[1]*1e6, wb/med[1]/1e9, med[0]/med[1]);
+ }
+ return 0;
+}
diff --git a/c/tests/test_int_kernel_exact.c b/c/tests/test_int_kernel_exact.c
index aa620173..25318a61 100644
--- a/c/tests/test_int_kernel_exact.c
+++ b/c/tests/test_int_kernel_exact.c
@@ -117,11 +117,21 @@ int main(void){
* float (per-gruppo: int esatto -> fmaf con la scala; poi * sx). Il kernel
* e' opt-in e non-bit-identico al f32 a gruppi; contro il SUO riferimento
* deve invece essere esatto al bit su ogni ISA. Copre gs=64 e gs=128
- * (bpg=2) e una coda I%gs!=0. */
+ * (bpg=2), una coda I%gs!=0, e le S che accendono le forme multi-riga:
+ * S=3 (solo per-riga), S=6/9 (tile 1x4 + resto), S=18 (due s-tile AMX,
+ * 16+2, sui build AMX β€” il gate S>=AMX_S_MIN e' il default 8). O=136
+ * lascia un resto O%16 cosi' il ramo AMX consegna anche la coda di
+ * output al path vettoriale. */
{
- int cases[][2]={{64,2048},{128,2048},{64,2000}};
- for(int t=0;t<3;t++){
- int gs=cases[t][0], I=cases[t][1], O=128, S=3;
+#ifdef COLI_HAVE_AMX_I4P
+ /* Emulator runs (Intel SDE) cannot take the OS tile-arming path; the
+ * arming is engine POLICY, the tile math is what this gate proves, so
+ * the test may force the state to reach the AMX kernel under SDE. */
+ if(getenv("COLI_TEST_FORCE_AMX")){ coli_amx_state=1; }
+#endif
+ int cases[][3]={{64,2048,3},{64,2048,9},{128,2048,18},{64,2000,6},{64,128,18}};
+ for(int t=0;t<5;t++){
+ int gs=cases[t][0], I=cases[t][1], O=136, S=cases[t][2];
int rb=(I+1)/2, ng=(I+gs-1)/gs;
uint8_t *qp=malloc((size_t)O*rb); for(size_t i=0;i<(size_t)O*rb;i++) qp[i]=(uint8_t)rnd();
uint8_t *ql=malloc((size_t)O*rb); memcpy(ql,qp,(size_t)O*rb); planarize_i4(ql,O,I);
diff --git a/docs/ENVIRONMENT.md b/docs/ENVIRONMENT.md
index b3d36252..9aecd8b8 100644
--- a/docs/ENVIRONMENT.md
+++ b/docs/ENVIRONMENT.md
@@ -117,7 +117,9 @@ Format: `VAR` β€” default β€” effect.
| `COLI_NO_FUSED_PAIR` | `0` (off) | `=1` disables the fused-pair matmul kernel. |
| `DISK_SPLIT` | `0` (off) | `=1` splits the reported disk-load time across the draft/absorb/forward phases in stats. |
| `I4S` | per-ISA (`1` on AVX-512-VNNI / NEON-dotprod, `2` elsewhere) | Engage the int4 `IDOT` kernel for batch `S>=<n>`. `I4S=1` turns IDOT on at decode too: int8-quantized activations on expert matmuls β€” **not bit-identical** to the f32 decode path (measured 0.39% of scale on the gate output; the same numerics prefill already uses at `S>=2`, and the shipped default on AVX-512-VNNI, measured +5.5% end-to-end there). Attention projections always stay exact regardless. A default flip on AVX-VNNI awaits the quality ablation. |
-| `IDOT_GS` | `0` (off) | **Opt-in** grouped planar IDOT for `fmt=4` (gs64/gs128) tensors: int8 activations with the K1 plane layout, one integer dot per scale group. Same numerics family as `I4S=1` β€” not bit-identical to the f32 grouped kernel, hence off until the ablation. Requires the planar family (AVX2 build, no GPU backend, no `XEXP`). Activation prints `[K1b]` once. |
+| `IDOT_GS` | `0` (off) | **Opt-in** grouped planar IDOT for `fmt=4` (gs64/gs128) tensors: int8 activations with the K1 plane layout, one integer dot per scale group. Same numerics family as `I4S=1` β€” not bit-identical to the f32 grouped kernel, hence off until the ablation. Requires the planar family (AVX2 or AVX-512 build β€” on AVX-512 only the fmt=4 tensors planarize, fmt=2 keeps the pair layout β€” no GPU backend, no `XEXP`). Multi-row calls (prefill batch-union, serve-mux decode) take a 1Γ—4 row tile that pays each weight block's unpack once per 4 rows; on AVX-512-VNNI whole 64-element groups go through single `vpdpbusd` zmm ops. All shapes are bit-identical to each other and to the pure-C reference (integer group dots, same per-row fmaf order). Activation prints `[K1b]` once. |
+| `AMX` | `1` (on where armable) | `=0` disables the K1c AMX int8 tile kernel inside the `IDOT_GS=1` family (Sapphire Rapids+; Linux arms tile state via `ARCH_REQ_XCOMP_PERM`, Windows 11 via `EnableProcessOptionalXStateFeatures`; other OSes fail closed). With gs a multiple of 64, one `tdpbssd` tile-multiply covers a scale group for 16 output rows Γ— up to 16 activation rows; bit-identical to the vector K1b path. Arming prints `[K1c]` once. |
+| `AMX_S_MIN` | `8` | Row threshold for the AMX tile kernel: below it the B-tile unpack does not amortize and the vector 1Γ—4 tile is the better kernel. Measure on your host β€” the break-even depends on cache level and core count. |
| `SPEC_PIN` | `1` (on) | Speculation gate mode. `0` reverts to the legacy S-dependent speculation gates (#163). |
| `COLI_RAM_OVERCOMMIT` | off | `=1` overrides the "projected peak > MemAvailable β†’ exit(2)" guard so a run that risks kernel OOM-kill is allowed to proceed. |