Run frontier MoE models on hardware you already own — pure C, zero deps, experts streamed from disk. Tiny engine, immense model. 🐦
C
perf(dsv4): decode fp8 attention weights with SIMD, not per-element gather
TheOptikd45e3d246c5b
diff --git a/c/deepseek_v4.c b/c/deepseek_v4.cindex 1ec6cf9e..de9bad25 100644--- a/c/deepseek_v4.c+++ b/c/deepseek_v4.c@@ -15188,6 +15188,32 @@ static int fp8_matvec_validate(const ColiTensorView *weight) {/* Compute CPU su attivazione GIA' qdq (estratto invariato da matvec_ref).* EN: CPU compute on an already-qdq'd activation, extracted verbatim. */+#ifdef __AVX2__+/* Branchless SIMD decode of 8 E4M3FN codes -> 8 f32, byte-identical to+ * coli_e4m3fn_decode for all 256 inputs (exhaustively verified). Replaces a+ * slow per-element _mm256_i32gather_ps from a 256-float LUT. E4M3FN values are+ * exact, so the f32 bit pattern is built directly: normals via integer field+ * assembly, subnormals as (float)mantissa*2^-9 (exact), NaN (code&0x7F==0x7F)+ * as canonical qNaN overwriting the sign. */+static inline __m256 v4_fp8_decode8(__m256i codes) {+ __m256i man = _mm256_and_si256(codes, _mm256_set1_epi32(7));+ __m256i exp = _mm256_and_si256(_mm256_srli_epi32(codes, 3), _mm256_set1_epi32(0xF));+ __m256i sgn = _mm256_slli_epi32(_mm256_srli_epi32(codes, 7), 31);+ __m256i nbits = _mm256_or_si256(+ _mm256_slli_epi32(_mm256_add_epi32(exp, _mm256_set1_epi32(120)), 23),+ _mm256_slli_epi32(man, 20));+ __m256 nval = _mm256_castsi256_ps(nbits);+ __m256 sval = _mm256_mul_ps(_mm256_cvtepi32_ps(man), _mm256_set1_ps(ldexpf(1.0f, -9)));+ __m256 is_sub = _mm256_castsi256_ps(_mm256_cmpeq_epi32(exp, _mm256_setzero_si256()));+ __m256i sbits = _mm256_or_si256(+ _mm256_castps_si256(_mm256_blendv_ps(nval, sval, is_sub)), sgn);+ __m256i is_nan = _mm256_cmpeq_epi32(+ _mm256_and_si256(codes, _mm256_set1_epi32(0x7F)), _mm256_set1_epi32(0x7F));+ return _mm256_castsi256_ps(+ _mm256_blendv_epi8(sbits, _mm256_set1_epi32(0x7FC00000), is_nan));+}+#endif+static int fp8_matvec_compute(float *output, const ColiTensorView *weight,const float *activation) {size_t rows = (size_t)weight->rows;@@ -15199,9 +15225,6 @@ static int fp8_matvec_compute(float *output, const ColiTensorView *weight,if (rows % 8) {return -1;}- float fp8[256];- for (int code = 0; code < 256; code++)- fp8[code] = coli_e4m3fn_decode((uint8_t)code);const uint8_t *data = weight->data;const float *scales = weight->scales;#pragma omp parallel for schedule(static)@@ -15216,7 +15239,7 @@ static int fp8_matvec_compute(float *output, const ColiTensorView *weight,__m128i bytes = _mm_loadl_epi64((const __m128i *)(data +((size_t)tile * columns + column) * 8));__m256i codes = _mm256_cvtepu8_epi32(bytes);- __m256 values = _mm256_i32gather_ps(fp8, codes, 4);+ __m256 values = v4_fp8_decode8(codes);__m256 x = _mm256_set1_ps(activation[column]);sum = _mm256_add_ps(sum, _mm256_mul_ps(_mm256_mul_ps(x, values), scale));diff --git a/c/tools/verify_fp8_decode.c b/c/tools/verify_fp8_decode.cnew file mode 100644index 00000000..fe16590d--- /dev/null+++ b/c/tools/verify_fp8_decode.c@@ -0,0 +1,57 @@+#include <immintrin.h>+#include <stdint.h>+#include <stdio.h>+#include <string.h>+#include <math.h>++/* reference: exact copy of coli_e4m3fn_decode (deepseek_v4.c:14774) */+static float ref_decode(uint8_t value) {+ int sign = value >> 7, exponent = (value >> 3) & 15, mantissa = value & 7;+ if (exponent == 15 && mantissa == 7) return NAN;+ float number;+ if (!exponent) number = ldexpf((float)mantissa, -9);+ else number = ldexpf(1.0f + (float)mantissa / 8.0f, exponent - 7);+ return sign ? -number : number;+}++/* candidate: branchless SIMD decode of 8 e4m3fn codes -> 8 f32, aiming bit-exact */+static inline __m256 fp8_decode8(__m256i codes) {+ __m256i man = _mm256_and_si256(codes, _mm256_set1_epi32(7));+ __m256i exp = _mm256_and_si256(_mm256_srli_epi32(codes, 3), _mm256_set1_epi32(0xF));+ __m256i sgn = _mm256_slli_epi32(_mm256_srli_epi32(codes, 7), 31); /* bit7 -> bit31 */+ /* normal magnitude: (exp+120)<<23 | man<<20 */+ __m256i nbits = _mm256_or_si256(+ _mm256_slli_epi32(_mm256_add_epi32(exp, _mm256_set1_epi32(120)), 23),+ _mm256_slli_epi32(man, 20));+ __m256 nval = _mm256_castsi256_ps(nbits);+ /* subnormal magnitude (exp==0): (float)man * 2^-9 (exact) */+ __m256 sval = _mm256_mul_ps(_mm256_cvtepi32_ps(man), _mm256_set1_ps(ldexpf(1.0f, -9)));+ __m256 is_sub = _mm256_castsi256_ps(_mm256_cmpeq_epi32(exp, _mm256_setzero_si256()));+ __m256 mag = _mm256_blendv_ps(nval, sval, is_sub);+ __m256i sbits = _mm256_or_si256(_mm256_castps_si256(mag), sgn); /* apply sign */+ /* NaN: (code & 0x7F) == 0x7F -> canonical qNaN, overwrites sign */+ __m256i is_nan = _mm256_cmpeq_epi32(_mm256_and_si256(codes, _mm256_set1_epi32(0x7F)),+ _mm256_set1_epi32(0x7F));+ __m256i fbits = _mm256_blendv_epi8(sbits, _mm256_set1_epi32(0x7FC00000), is_nan);+ return _mm256_castsi256_ps(fbits);+}++int main(void) {+ /* what bits does NAN actually compile to here? */+ float n = NAN; uint32_t nb; memcpy(&nb, &n, 4);+ printf("platform NAN bits = 0x%08x (my const = 0x7FC00000)\n", nb);+ int mism = 0;+ for (int base = 0; base < 256; base += 8) {+ int32_t c[8]; for (int i = 0; i < 8; i++) c[i] = base + i;+ __m256i codes = _mm256_loadu_si256((const __m256i *)c);+ float out[8]; _mm256_storeu_ps(out, fp8_decode8(codes));+ for (int i = 0; i < 8; i++) {+ float r = ref_decode((uint8_t)(base + i));+ uint32_t rb, ob; memcpy(&rb, &r, 4); memcpy(&ob, &out[i], 4);+ if (rb != ob) { printf("MISMATCH code=%3d ref=0x%08x(%g) mine=0x%08x(%g)\n",+ base + i, rb, r, ob, out[i]); mism++; }+ }+ }+ printf(mism ? "FAIL: %d mismatches\n" : "ALL 256 CODES BIT-EXACT\n", mism);+ return mism ? 1 : 0;+}