Skip to content

Run frontier MoE models on hardware you already own — pure C, zero deps, experts streamed from disk. Tiny engine, immense model. 🐦

C

quant: AVX2 matmul_e8 — fmt=6 was 92% of decode on a scalar kernel

Steve Markgrafe2c3d941c193
diff --git a/c/quant.h b/c/quant.h
index 505141f5..70fcf5c7 100644
--- a/c/quant.h
+++ b/c/quant.h
@@ -1200,6 +1200,52 @@ static void matmul_e8(float *y, const float *x, const uint8_t *q, const float *u
uint16_t dh; memcpy(&dh, blk+96, 2);
float d=e8_fp16_to_f32(dh);
int base=(int)(b*E8_QK);
+#ifdef __AVX2__
+ /* One 8-weight lane is exactly one AVX2 register, which is what the
+ * format's own shape suggests: a lane is two 4-dim grid rows (8
+ * contiguous codebook bytes) plus 8 signs. So instead of expanding a
+ * sub-block into a stack buffer and re-reading it, each lane is
+ * decoded straight into a register and FMA'd against x:
+ * - the 8 grid bytes widen with one vpmovzxbd,
+ * - the sign byte (7 stored bits + the parity-derived 8th) expands
+ * to 8 lane masks with an AND/CMPEQ against the bit-select vector
+ * and is applied as an XOR of the float sign bit — no branches,
+ * which is what made the scalar expansion expensive,
+ * - 0.5 (the grid's half-unit convention) folds into the sub-scale.
+ * Two accumulators over the 32 FMAs of a super-block keep this off a
+ * single dependency chain, and one horizontal add per 256 weights
+ * replaces one per 32. */
+ if(base+E8_QK<=I){
+ const __m256i sel=_mm256_setr_epi32(1,2,4,8,16,32,64,128);
+ const __m256i sgn=_mm256_set1_epi32((int)0x80000000u);
+ __m256 ac[2]={_mm256_setzero_ps(),_mm256_setzero_ps()};
+ for(int ib=0; ib<E8_QK/E8_SUB; ib++){
+ uint32_t word; memcpy(&word, blk+E8_QK/4+ib*4, 4);
+ float db=d*(0.5f+(float)((word>>28)&0xF))*0.5f;
+ __m256 vdb=_mm256_set1_ps(0.5f*db);
+ const uint8_t *ix=blk+ib*8;
+ int off=base+ib*E8_SUB;
+ for(int l=0;l<4;l++){
+ uint32_t sv=(word>>(7*l))&0x7Fu;
+ uint32_t s8=sv|((uint32_t)__builtin_parity(sv)<<7); /* odd parity closes the lane */
+ uint32_t g0,g1;
+ memcpy(&g0,e8_grid[ix[l*2+0]],4);
+ memcpy(&g1,e8_grid[ix[l*2+1]],4);
+ __m128i by=_mm_cvtsi64_si128((long long)((uint64_t)g0|((uint64_t)g1<<32)));
+ __m256 v=_mm256_mul_ps(_mm256_cvtepi32_ps(_mm256_cvtepu8_epi32(by)),vdb);
+ __m256i m=_mm256_cmpeq_epi32(_mm256_and_si256(_mm256_set1_epi32((int)s8),sel),sel);
+ v=_mm256_xor_ps(v,_mm256_castsi256_ps(_mm256_and_si256(m,sgn)));
+ ac[l&1]=_mm256_fmadd_ps(v,_mm256_loadu_ps(xs+off+l*8),ac[l&1]);
+ }
+ }
+ __m256 t=_mm256_add_ps(ac[0],ac[1]);
+ __m128 h=_mm_add_ps(_mm256_castps256_ps128(t),_mm256_extractf128_ps(t,1));
+ h=_mm_add_ps(h,_mm_movehl_ps(h,h));
+ h=_mm_add_ss(h,_mm_shuffle_ps(h,h,1));
+ acc+=_mm_cvtss_f32(h);
+ continue;
+ }
+#endif
for(int ib=0; ib<E8_QK/E8_SUB; ib++){
int off=base+ib*E8_SUB;
if(off>=I) break;