Skip to content

Run frontier MoE models on hardware you already own — pure C, zero deps, experts streamed from disk. Tiny engine, immense model. 🐦

C

perf(dsv4): decode fp8 attention weights with SIMD, not per-element gather

TheOptikd45e3d246c5b
diff --git a/c/deepseek_v4.c b/c/deepseek_v4.c
index 1ec6cf9e..de9bad25 100644
--- a/c/deepseek_v4.c
+++ b/c/deepseek_v4.c
@@ -15188,6 +15188,32 @@ static int fp8_matvec_validate(const ColiTensorView *weight) {
/* Compute CPU su attivazione GIA' qdq (estratto invariato da matvec_ref).
* EN: CPU compute on an already-qdq'd activation, extracted verbatim. */
+#ifdef __AVX2__
+/* Branchless SIMD decode of 8 E4M3FN codes -> 8 f32, byte-identical to
+ * coli_e4m3fn_decode for all 256 inputs (exhaustively verified). Replaces a
+ * slow per-element _mm256_i32gather_ps from a 256-float LUT. E4M3FN values are
+ * exact, so the f32 bit pattern is built directly: normals via integer field
+ * assembly, subnormals as (float)mantissa*2^-9 (exact), NaN (code&0x7F==0x7F)
+ * as canonical qNaN overwriting the sign. */
+static inline __m256 v4_fp8_decode8(__m256i codes) {
+ __m256i man = _mm256_and_si256(codes, _mm256_set1_epi32(7));
+ __m256i exp = _mm256_and_si256(_mm256_srli_epi32(codes, 3), _mm256_set1_epi32(0xF));
+ __m256i sgn = _mm256_slli_epi32(_mm256_srli_epi32(codes, 7), 31);
+ __m256i nbits = _mm256_or_si256(
+ _mm256_slli_epi32(_mm256_add_epi32(exp, _mm256_set1_epi32(120)), 23),
+ _mm256_slli_epi32(man, 20));
+ __m256 nval = _mm256_castsi256_ps(nbits);
+ __m256 sval = _mm256_mul_ps(_mm256_cvtepi32_ps(man), _mm256_set1_ps(ldexpf(1.0f, -9)));
+ __m256 is_sub = _mm256_castsi256_ps(_mm256_cmpeq_epi32(exp, _mm256_setzero_si256()));
+ __m256i sbits = _mm256_or_si256(
+ _mm256_castps_si256(_mm256_blendv_ps(nval, sval, is_sub)), sgn);
+ __m256i is_nan = _mm256_cmpeq_epi32(
+ _mm256_and_si256(codes, _mm256_set1_epi32(0x7F)), _mm256_set1_epi32(0x7F));
+ return _mm256_castsi256_ps(
+ _mm256_blendv_epi8(sbits, _mm256_set1_epi32(0x7FC00000), is_nan));
+}
+#endif
+
static int fp8_matvec_compute(float *output, const ColiTensorView *weight,
const float *activation) {
size_t rows = (size_t)weight->rows;
@@ -15199,9 +15225,6 @@ static int fp8_matvec_compute(float *output, const ColiTensorView *weight,
if (rows % 8) {
return -1;
}
- float fp8[256];
- for (int code = 0; code < 256; code++)
- fp8[code] = coli_e4m3fn_decode((uint8_t)code);
const uint8_t *data = weight->data;
const float *scales = weight->scales;
#pragma omp parallel for schedule(static)
@@ -15216,7 +15239,7 @@ static int fp8_matvec_compute(float *output, const ColiTensorView *weight,
__m128i bytes = _mm_loadl_epi64((const __m128i *)(data +
((size_t)tile * columns + column) * 8));
__m256i codes = _mm256_cvtepu8_epi32(bytes);
- __m256 values = _mm256_i32gather_ps(fp8, codes, 4);
+ __m256 values = v4_fp8_decode8(codes);
__m256 x = _mm256_set1_ps(activation[column]);
sum = _mm256_add_ps(sum, _mm256_mul_ps(
_mm256_mul_ps(x, values), scale));
diff --git a/c/tools/verify_fp8_decode.c b/c/tools/verify_fp8_decode.c
new file mode 100644
index 00000000..fe16590d
--- /dev/null
+++ b/c/tools/verify_fp8_decode.c
@@ -0,0 +1,57 @@
+#include <immintrin.h>
+#include <stdint.h>
+#include <stdio.h>
+#include <string.h>
+#include <math.h>
+
+/* reference: exact copy of coli_e4m3fn_decode (deepseek_v4.c:14774) */
+static float ref_decode(uint8_t value) {
+ int sign = value >> 7, exponent = (value >> 3) & 15, mantissa = value & 7;
+ if (exponent == 15 && mantissa == 7) return NAN;
+ float number;
+ if (!exponent) number = ldexpf((float)mantissa, -9);
+ else number = ldexpf(1.0f + (float)mantissa / 8.0f, exponent - 7);
+ return sign ? -number : number;
+}
+
+/* candidate: branchless SIMD decode of 8 e4m3fn codes -> 8 f32, aiming bit-exact */
+static inline __m256 fp8_decode8(__m256i codes) {
+ __m256i man = _mm256_and_si256(codes, _mm256_set1_epi32(7));
+ __m256i exp = _mm256_and_si256(_mm256_srli_epi32(codes, 3), _mm256_set1_epi32(0xF));
+ __m256i sgn = _mm256_slli_epi32(_mm256_srli_epi32(codes, 7), 31); /* bit7 -> bit31 */
+ /* normal magnitude: (exp+120)<<23 | man<<20 */
+ __m256i nbits = _mm256_or_si256(
+ _mm256_slli_epi32(_mm256_add_epi32(exp, _mm256_set1_epi32(120)), 23),
+ _mm256_slli_epi32(man, 20));
+ __m256 nval = _mm256_castsi256_ps(nbits);
+ /* subnormal magnitude (exp==0): (float)man * 2^-9 (exact) */
+ __m256 sval = _mm256_mul_ps(_mm256_cvtepi32_ps(man), _mm256_set1_ps(ldexpf(1.0f, -9)));
+ __m256 is_sub = _mm256_castsi256_ps(_mm256_cmpeq_epi32(exp, _mm256_setzero_si256()));
+ __m256 mag = _mm256_blendv_ps(nval, sval, is_sub);
+ __m256i sbits = _mm256_or_si256(_mm256_castps_si256(mag), sgn); /* apply sign */
+ /* NaN: (code & 0x7F) == 0x7F -> canonical qNaN, overwrites sign */
+ __m256i is_nan = _mm256_cmpeq_epi32(_mm256_and_si256(codes, _mm256_set1_epi32(0x7F)),
+ _mm256_set1_epi32(0x7F));
+ __m256i fbits = _mm256_blendv_epi8(sbits, _mm256_set1_epi32(0x7FC00000), is_nan);
+ return _mm256_castsi256_ps(fbits);
+}
+
+int main(void) {
+ /* what bits does NAN actually compile to here? */
+ float n = NAN; uint32_t nb; memcpy(&nb, &n, 4);
+ printf("platform NAN bits = 0x%08x (my const = 0x7FC00000)\n", nb);
+ int mism = 0;
+ for (int base = 0; base < 256; base += 8) {
+ int32_t c[8]; for (int i = 0; i < 8; i++) c[i] = base + i;
+ __m256i codes = _mm256_loadu_si256((const __m256i *)c);
+ float out[8]; _mm256_storeu_ps(out, fp8_decode8(codes));
+ for (int i = 0; i < 8; i++) {
+ float r = ref_decode((uint8_t)(base + i));
+ uint32_t rb, ob; memcpy(&rb, &r, 4); memcpy(&ob, &out[i], 4);
+ if (rb != ob) { printf("MISMATCH code=%3d ref=0x%08x(%g) mine=0x%08x(%g)\n",
+ base + i, rb, r, ob, out[i]); mism++; }
+ }
+ }
+ printf(mism ? "FAIL: %d mismatches\n" : "ALL 256 CODES BIT-EXACT\n", mism);
+ return mism ? 1 : 0;
+}