Run frontier MoE models on hardware you already own — pure C, zero deps, experts streamed from disk. Tiny engine, immense model. 🐦
C
perf(dsv4): specialize batch-one FP4 matvecs
JustVugg13a4282c3e65
diff --git a/c/deepseek_v4.c b/c/deepseek_v4.cindex 1ec6cf9e..531f2797 100644--- a/c/deepseek_v4.c+++ b/c/deepseek_v4.c@@ -14752,6 +14752,7 @@ int coli_v4_config_load(ColiDeepSeekV4Config *config, const char *model_dir,#include <float.h>#include <math.h>+#include <pthread.h>#include <stdlib.h>#include <string.h>#ifdef __AVX2__@@ -14763,6 +14764,24 @@ float coli_e8m0_decode(uint8_t value) {return ldexpf(1.0f, (int)value - 127);}+/* Every FP4 matvec used to rebuild this table locally, including 255 calls+ * to ldexpf. Decode performs three matvecs per routed expert, so batch-one+ * decode paid that setup hundreds of thousands of times per response. A+ * process-wide immutable table keeps the exact decoder as its single source+ * of truth while making both scalar and batched kernels reuse the result. */+static float coli_e8m0_lut[256];+static pthread_once_t coli_e8m0_lut_once = PTHREAD_ONCE_INIT;++static void coli_e8m0_lut_initialize(void) {+ for (int value = 0; value < 256; value++)+ coli_e8m0_lut[value] = coli_e8m0_decode((uint8_t)value);+}++const float *coli_e8m0_table(void) {+ pthread_once(&coli_e8m0_lut_once, coli_e8m0_lut_initialize);+ return coli_e8m0_lut;+}+float coli_e2m1_decode(uint8_t nibble) {static const float values[16] = {0.0f, 0.5f, 1.0f, 1.5f, 2.0f, 3.0f, 4.0f, 6.0f,@@ -14996,7 +15015,17 @@ int coli_v4_qdq_scratch(size_t activation_count, size_t scales_count,void coli_fp4_matmul_batch_rows16_order(float *y, const uint8_t *q4,const uint8_t *e8s, const float *x,int S, int I, int O) {+ /* Decode is always batch-one. Keep it on the register-resident kernel;+ * the runtime-sized accumulator arrays below necessarily spill on AVX2.+ * Prefill still shares decoded weights whenever two or more positions+ * route to the same expert. */+ if (S == 1) {+ coli_fp4_matvec_rows16_order(y, q4, e8s, x, I, O);+ return;+ }int rb = I / 2, ng = I / 32;+ float e8lut[256];+ memcpy(e8lut, coli_e8m0_table(), sizeof(e8lut));#ifdef __AVX2__/* Nibble decode borrows matmul_mxfp4's trick — doubled e2m1 values are* exact int8, one pshufb decodes a 16-byte row into 32 codes — but the@@ -15006,8 +15035,6 @@ void coli_fp4_matmul_batch_rows16_order(float *y, const uint8_t *q4,* rows16 kernels. Rows decode row-major (streaming loads), then 8x8* transposes turn them column-major so 16 rows ride two 8-lane vectors* down the column loop in the rows16 per-row order: mul, mul, add. */- float e8lut[256];- for (int v = 0; v < 256; v++) e8lut[v] = coli_e8m0_decode((uint8_t)v);#pragma omp parallel for schedule(static)for (int64_t tile = 0; tile < O / 16; tile++) {const __m128i lut2 = _mm_setr_epi8(0,1,2,3,4,6,8,12,0,-1,-2,-3,-4,-6,-8,-12);@@ -15106,7 +15133,7 @@ void coli_fp4_matmul_batch_rows16_order(float *y, const uint8_t *q4,uint8_t byte = w[c >> 1];float wv = coli_e2m1_decode((c & 1) ? (uint8_t)(byte >> 4): (uint8_t)(byte & 0xF));- float scale = coli_e8m0_decode(scl[c / 32]);+ float scale = e8lut[scl[c / 32]];for (int s = 0; s < S; s++) {float t = x[(int64_t)s * I + c] * wv;t = t * scale;@@ -15121,7 +15148,122 @@ void coli_fp4_matmul_batch_rows16_order(float *y, const uint8_t *q4,void coli_fp4_matvec_rows16_order(float *y, const uint8_t *q4,const uint8_t *e8s, const float *x,int I, int O) {- coli_fp4_matmul_batch_rows16_order(y, q4, e8s, x, 1, I, O);+ int rb = I / 2, ng = I / 32;+ float e8lut[256];+ memcpy(e8lut, coli_e8m0_table(), sizeof(e8lut));+#ifdef __AVX2__+ /* Batch-one has its own loop so sum0/sum1 remain YMM registers for the+ * whole row tile. Do not fold this back into the runtime-S kernel: even+ * when its caller passes one, the OpenMP worker cannot specialize S and+ * reloads/stores the accumulator arrays in every column iteration. */+ #pragma omp parallel for schedule(static)+ for (int64_t tile = 0; tile < O / 16; tile++) {+ const __m128i lut2 = _mm_setr_epi8(+ 0,1,2,3,4,6,8,12,0,-1,-2,-3,-4,-6,-8,-12);+ const __m128i m4 = _mm_set1_epi8(0x0F);+ const __m256 half = _mm256_set1_ps(0.5f);+ __m256 sum0 = _mm256_setzero_ps(), sum1 = _mm256_setzero_ps();+ for (int base = 0; base < I; base += 32) {+ float sc[16], tmp[2][32][8];+ for (int half_rows = 0; half_rows < 2; half_rows++) {+ __m256 rowv[8][4];+ for (int r8 = 0; r8 < 8; r8++) {+ int64_t row = tile * 16 + half_rows * 8 + r8;+ sc[half_rows * 8 + r8] =+ e8lut[e8s[row * ng + base / 32]];+ __m128i by = _mm_loadu_si128(+ (const __m128i *)(q4 + row * rb + base / 2));+ __m128i lo = _mm_and_si128(by, m4);+ __m128i hi = _mm_and_si128(_mm_srli_epi16(by, 4), m4);+ __m128i n0 = _mm_shuffle_epi8(+ lut2, _mm_unpacklo_epi8(lo, hi));+ __m128i n1 = _mm_shuffle_epi8(+ lut2, _mm_unpackhi_epi8(lo, hi));+ rowv[r8][0] = _mm256_mul_ps(_mm256_cvtepi32_ps(+ _mm256_cvtepi8_epi32(n0)), half);+ rowv[r8][1] = _mm256_mul_ps(_mm256_cvtepi32_ps(+ _mm256_cvtepi8_epi32(_mm_srli_si128(n0, 8))), half);+ rowv[r8][2] = _mm256_mul_ps(_mm256_cvtepi32_ps(+ _mm256_cvtepi8_epi32(n1)), half);+ rowv[r8][3] = _mm256_mul_ps(_mm256_cvtepi32_ps(+ _mm256_cvtepi8_epi32(_mm_srli_si128(n1, 8))), half);+ }+ for (int cb = 0; cb < 4; cb++) {+ __m256 blk[8];+ for (int r8 = 0; r8 < 8; r8++)+ blk[r8] = rowv[r8][cb];+ __m256 t0 = _mm256_unpacklo_ps(blk[0], blk[1]);+ __m256 t1 = _mm256_unpackhi_ps(blk[0], blk[1]);+ __m256 t2 = _mm256_unpacklo_ps(blk[2], blk[3]);+ __m256 t3 = _mm256_unpackhi_ps(blk[2], blk[3]);+ __m256 t4 = _mm256_unpacklo_ps(blk[4], blk[5]);+ __m256 t5 = _mm256_unpackhi_ps(blk[4], blk[5]);+ __m256 t6 = _mm256_unpacklo_ps(blk[6], blk[7]);+ __m256 t7 = _mm256_unpackhi_ps(blk[6], blk[7]);+ __m256 s0 = _mm256_shuffle_ps(+ t0, t2, _MM_SHUFFLE(1,0,1,0));+ __m256 s1 = _mm256_shuffle_ps(+ t0, t2, _MM_SHUFFLE(3,2,3,2));+ __m256 s2 = _mm256_shuffle_ps(+ t1, t3, _MM_SHUFFLE(1,0,1,0));+ __m256 s3 = _mm256_shuffle_ps(+ t1, t3, _MM_SHUFFLE(3,2,3,2));+ __m256 s4 = _mm256_shuffle_ps(+ t4, t6, _MM_SHUFFLE(1,0,1,0));+ __m256 s5 = _mm256_shuffle_ps(+ t4, t6, _MM_SHUFFLE(3,2,3,2));+ __m256 s6 = _mm256_shuffle_ps(+ t5, t7, _MM_SHUFFLE(1,0,1,0));+ __m256 s7 = _mm256_shuffle_ps(+ t5, t7, _MM_SHUFFLE(3,2,3,2));+ _mm256_storeu_ps(tmp[half_rows][cb * 8 + 0],+ _mm256_permute2f128_ps(s0, s4, 0x20));+ _mm256_storeu_ps(tmp[half_rows][cb * 8 + 1],+ _mm256_permute2f128_ps(s1, s5, 0x20));+ _mm256_storeu_ps(tmp[half_rows][cb * 8 + 2],+ _mm256_permute2f128_ps(s2, s6, 0x20));+ _mm256_storeu_ps(tmp[half_rows][cb * 8 + 3],+ _mm256_permute2f128_ps(s3, s7, 0x20));+ _mm256_storeu_ps(tmp[half_rows][cb * 8 + 4],+ _mm256_permute2f128_ps(s0, s4, 0x31));+ _mm256_storeu_ps(tmp[half_rows][cb * 8 + 5],+ _mm256_permute2f128_ps(s1, s5, 0x31));+ _mm256_storeu_ps(tmp[half_rows][cb * 8 + 6],+ _mm256_permute2f128_ps(s2, s6, 0x31));+ _mm256_storeu_ps(tmp[half_rows][cb * 8 + 7],+ _mm256_permute2f128_ps(s3, s7, 0x31));+ }+ }+ __m256 sc0 = _mm256_loadu_ps(sc);+ __m256 sc1 = _mm256_loadu_ps(sc + 8);+ for (int c = 0; c < 32; c++) {+ __m256 xv = _mm256_set1_ps(x[base + c]);+ sum0 = _mm256_add_ps(sum0, _mm256_mul_ps(+ _mm256_mul_ps(xv, _mm256_loadu_ps(tmp[0][c])), sc0));+ sum1 = _mm256_add_ps(sum1, _mm256_mul_ps(+ _mm256_mul_ps(xv, _mm256_loadu_ps(tmp[1][c])), sc1));+ }+ }+ _mm256_storeu_ps(y + tile * 16, sum0);+ _mm256_storeu_ps(y + tile * 16 + 8, sum1);+ }+#else+ #pragma omp parallel for schedule(static)+ for (int o = 0; o < O; o++) {+ const uint8_t *w = q4 + (int64_t)o * rb;+ const uint8_t *scl = e8s + (int64_t)o * ng;+ float sum = 0.0f;+ for (int c = 0; c < I; c++) {+ uint8_t byte = w[c >> 1];+ float wv = coli_e2m1_decode((c & 1)+ ? (uint8_t)(byte >> 4) : (uint8_t)(byte & 0xF));+ float t = x[c] * wv;+ t = t * e8lut[scl[c / 32]];+ sum = sum + t;+ }+ y[o] = sum;+ }+#endif}int coli_fp4_matvec_ref(float *output, const ColiTensorView *weight,@@ -15631,6 +15773,7 @@ int coli_fp4_matmul_batch_ref(float *outputs, const ColiTensorView *weight,#endif#include <stdint.h>#include <stdlib.h>+#include <string.h>static int source_valid(const ColiTensorView *weight) {if (!weight || weight->format != COLI_TENSOR_FP4_NATIVE_BLOCK ||@@ -15678,7 +15821,7 @@ int coli_fp4_pack_rows16_v10(unsigned char *packed_data,static void decode_tables(__m512 *fp4_table, float e8[256]) {float fp4[16];for (int i = 0; i < 16; i++) fp4[i] = coli_e2m1_decode((uint8_t)i);- for (int i = 0; i < 256; i++) e8[i] = coli_e8m0_decode((uint8_t)i);+ memcpy(e8, coli_e8m0_table(), 256 * sizeof(*e8));*fp4_table = _mm512_loadu_ps(fp4);}@@ -15706,8 +15849,7 @@ typedef struct Avx2Rows16Tables {static void avx2_rows16_tables(Avx2Rows16Tables *tables) {for (int i = 0; i < 16; i++)tables->fp4[i] = coli_e2m1_decode((uint8_t)i);- for (int i = 0; i < 256; i++)- tables->e8[i] = coli_e8m0_decode((uint8_t)i);+ memcpy(tables->e8, coli_e8m0_table(), sizeof(tables->e8));}static inline void avx2_decode_rows16(__m256 values[2],@@ -15756,7 +15898,7 @@ typedef struct NeonRows16Tables {static void neon_rows16_tables(NeonRows16Tables *tables) {float fp4[16];for (int i = 0; i < 16; i++) fp4[i] = coli_e2m1_decode((uint8_t)i);- for (int i = 0; i < 256; i++) tables->e8[i] = coli_e8m0_decode((uint8_t)i);+ memcpy(tables->e8, coli_e8m0_table(), sizeof(tables->e8));const unsigned char *bytes = (const unsigned char *)fp4;for (int group = 0; group < 4; group++)tables->fp4_bytes.val[group] = vld1q_u8(bytes + 16 * group);diff --git a/c/native_quant.h b/c/native_quant.hindex 89ba0536..01b137ce 100644--- a/c/native_quant.h+++ b/c/native_quant.h@@ -11,6 +11,8 @@ extern "C" {#endiffloat coli_e8m0_decode(uint8_t value);+/* Process-wide immutable decode table shared by split native-quant units. */+const float *coli_e8m0_table(void);float coli_e2m1_decode(uint8_t nibble);float coli_e4m3fn_decode(uint8_t value);uint8_t coli_e4m3fn_encode(float value);diff --git a/c/tests/test_deepseek_v4.c b/c/tests/test_deepseek_v4.cindex c92a9ec3..8c85992e 100644--- a/c/tests/test_deepseek_v4.c+++ b/c/tests/test_deepseek_v4.c@@ -285,6 +285,17 @@ static int test_expert_batch(void) {}return 1;}+ /* The public batch entry keeps batch-one on the dedicated matvec path;+ * protect its exact-output contract alongside the separate perf check. */+ float scalar_matrix[INTERMEDIATE], batch_one[INTERMEDIATE];+ coli_fp4_matvec_rows16_order(+ scalar_matrix, gate_data, gate_scale, inputs, DIMENSION, INTERMEDIATE);+ coli_fp4_matmul_batch_rows16_order(+ batch_one, gate_data, gate_scale, inputs, 1, DIMENSION, INTERMEDIATE);+ if (memcmp(scalar_matrix, batch_one, sizeof(scalar_matrix)) != 0) {+ fputs("FP4 batch-one dispatch diverged from scalar matvec\n", stderr);+ return 1;+ }uint64_t calls = coli_v4_test_fp4_batch_calls;if (coli_v4_expert_forward_batch_ref(batched, &expert, inputs, route_weights, 0, 10.0f) == 0 ||diff --git a/c/tests/test_native_quant.c b/c/tests/test_native_quant.cindex 0e05f67a..a451f453 100644--- a/c/tests/test_native_quant.c+++ b/c/tests/test_native_quant.c@@ -23,6 +23,13 @@ int main(void) {!close_enough(coli_e8m0_decode(0x7f), 1.0f) ||!close_enough(coli_e8m0_decode(0x80), 2.0f) ||!isnan(coli_e8m0_decode(0xff))) return 1;+ const float *e8m0 = coli_e8m0_table();+ if (!e8m0 || e8m0 != coli_e8m0_table()) return 1;+ for (int code = 0; code < 255; code++) {+ float decoded = coli_e8m0_decode((uint8_t)code);+ if (memcmp(&e8m0[code], &decoded, sizeof(decoded)) != 0) return 1;+ }+ if (!isnan(e8m0[255])) return 1;static const float representable[] = {0.0f, 0.001953125f, 0.5f, 1.0f, 1.5f, 6.0f,