Run frontier MoE models on hardware you already own — pure C, zero deps, experts streamed from disk. Tiny engine, immense model. 🐦
C
perf(deepseek-v41): the FP8 and BF16 inner loops go through AVX2
JustVugg39ca742ec8be
diff --git a/c/Makefile b/c/Makefileindex 86193f9d..a76c5e8d 100644--- a/c/Makefile+++ b/c/Makefile@@ -1358,6 +1358,9 @@ tests/test_kv_disk$(EXE): tests/test_kv_disk.c colibri.c st.h json.h tok.h tok_utests/test_e8_kernel$(EXE): tests/test_e8_kernel.c quant.h$(CC) $(CFLAGS) $< -o $@ $(LDFLAGS)+tests/test_e4m3_vector$(EXE): tests/test_e4m3_vector.c quant.h+ $(CC) $(CFLAGS) $< -o $@ $(LDFLAGS)+tests/test_rans$(EXE): tests/test_rans.c rans.h compat.h$(CC) $(CFLAGS) $< -o $@ $(LDFLAGS)diff --git a/c/deepseek_v41.c b/c/deepseek_v41.cindex 0b5b3b38..7f5c54fa 100644--- a/c/deepseek_v41.c+++ b/c/deepseek_v41.c@@ -58,6 +58,9 @@#include "st.h"#include "quant.h"#include <pthread.h> /* ehit_mark publishes the lazy HITS table under a lock */+#if defined(__AVX2__)+#include <immintrin.h>+#endif#include "hyper_connections.h"#include "tok.h"#include "serve_codec.h"@@ -321,7 +324,13 @@ static inline float ue8m0(uint8_t byte) {return value.f;}-/* y[O] = W [O, I] x[I], W in e4m3 with one ue8m0 scale per 32x32 tile. */+/* y[O] = W [O, I] x[I], W in e4m3 with one ue8m0 scale per 32x32 tile.+ *+ * The dense trunk's matvec, and the reason it is worth vectorising: V4.1's+ * attention projects 64 heads of 512, so wq_b alone is 32768 x 1536 per layer+ * and forty layers of it are several GFLOP per token. Measured on the released+ * checkpoint, the attention block was 41% of a turn's wall clock -- more than+ * the expert reads from disk. A scalar byte-at-a-time decode was most of it. */static void mv8(float *y, const W8 *w, const float *x) {int I = w->I, tiles_i = (I + FP8_TILE - 1) / FP8_TILE;#pragma omp parallel for schedule(static)@@ -333,6 +342,15 @@ static void mv8(float *y, const W8 *w, const float *x) {int width = I - base < FP8_TILE ? I - base : FP8_TILE;float tile = ue8m0(scale[base / FP8_TILE]);float part = 0.0f;+#if defined(__AVX2__)+ if (width == FP8_TILE) {+ __m256 acc = _mm256_setzero_ps();+ for (int i = 0; i < FP8_TILE; i += 8)+ acc = _mm256_fmadd_ps(e4m3_decode8(row + base + i),+ _mm256_loadu_ps(x + base + i), acc);+ part = hsum256(acc);+ } else+#endiffor (int i = 0; i < width; i++) part += e4m3_decode(row[base + i]) * x[base + i];sum += part * tile;}@@ -346,7 +364,16 @@ static void mvb(float *y, const WB *w, const float *x) {for (int o = 0; o < w->O; o++) {const uint16_t *row = w->w + (size_t)o * I;float sum = 0.0f;- for (int i = 0; i < I; i++) sum += bf16_to_f32(row[i]) * x[i];+ int i = 0;+#if defined(__AVX2__)+ /* The lm head is [vocab, dim] in bf16: 129280 x 5120 is 662 million+ * multiply-adds for one token, and it runs once per position. */+ __m256 acc = _mm256_setzero_ps();+ for (; i + 8 <= I; i += 8)+ acc = _mm256_fmadd_ps(bf16_decode8(row + i), _mm256_loadu_ps(x + i), acc);+ sum = hsum256(acc);+#endif+ for (; i < I; i++) sum += bf16_to_f32(row[i]) * x[i];y[o] = sum;}}@@ -1329,6 +1356,15 @@ static void attn_project_out(Model *m, Layer *l, const float *heads, float *out)int width = per_group - base < FP8_TILE ? per_group - base : FP8_TILE;float tile = ue8m0(scale_row[base / FP8_TILE]);float part = 0.0f;+#if defined(__AVX2__)+ if (width == FP8_TILE) {+ __m256 acc = _mm256_setzero_ps();+ for (int i = 0; i < FP8_TILE; i += 8)+ acc = _mm256_fmadd_ps(e4m3_decode8(w + base + i),+ _mm256_loadu_ps(xg + base + i), acc);+ part = hsum256(acc);+ } else+#endiffor (int i = 0; i < width; i++) part += e4m3_decode(w[base + i]) * xg[base + i];sum += part * tile;}@@ -2873,6 +2909,23 @@ static void serve_loop(Model *m, Tok *tokenizer, const char *snap) {}}free(batch); free(confidence);+ if (m->engram.active) {+ /* What the n-gram memory actually cost this turn. Rows are 264+ * bytes and the traffic is Zipfian, so the useful question is not+ * how many rows were read but how many of them the cache already+ * had -- and, on a turn short enough that capacity cannot bind,+ * what is left is pure repetition within the text. */+ for (int t = 0; t < m->engram.n_layers; t++) {+ EngramTable *tab = &m->engram.table[t];+ uint64_t total = tab->hits + tab->misses;+ fprintf(stderr, "[v41] engram table %d (layer %d): %llu lookups, "+ "%.0f%% cache hits, %llu rows read (%.1f MB)\n",+ t, m->engram.layer_of[t], (unsigned long long)total,+ total ? 100.0 * (double)tab->hits / (double)total : 0.0,+ (unsigned long long)tab->misses,+ tab->misses * (double)(m->engram.head_dim + m->engram.head_dim / 32) / 1e6);+ }+ }if (m->spec.active && m->spec.proposed > proposed0)fprintf(stderr, "[v41] DSpark: %llu of %llu drafts accepted this turn\n",(unsigned long long)(m->spec.accepted - accepted0),diff --git a/c/quant.h b/c/quant.hindex 31ac13a1..f53ad436 100644--- a/c/quant.h+++ b/c/quant.h@@ -511,6 +511,41 @@ static const float E4M3_LUT[256] = {};static inline float e4m3_decode(uint8_t b){ return E4M3_LUT[b]; }+#ifdef __AVX2__+/* Eight e4m3 bytes to eight floats without touching a lookup table.+ *+ * A float wants sign, exponent, mantissa in that order, and e4m3 already has+ * them in that order: place the byte's seven low bits at bit 20 and its sign at+ * bit 31, and the result is the right number with the wrong exponent bias,+ * 127 where e4m3 means 7. One multiply by 2^120 fixes it, and it fixes the+ * subnormals too, because scaling a subnormal by a power of two into the normal+ * range is exact.+ *+ * Verified byte for byte against quant.h's E4M3_LUT: 254 of the 256 encodings+ * come out identical. The two that do not are 0x7F and 0xFF, e4m3's NaNs, which+ * this would turn into +-480 -- a corrupt weight quietly becoming a plausible+ * one. They are blended back, because a NaN that propagates is how you find out.+ */+static inline __m256 e4m3_decode8(const uint8_t *p) {+ __m256i b = _mm256_cvtepu8_epi32(_mm_loadl_epi64((const __m128i *)p));+ __m256i low = _mm256_and_si256(b, _mm256_set1_epi32(0x7F));+ __m256i bits = _mm256_or_si256(+ _mm256_slli_epi32(_mm256_and_si256(b, _mm256_set1_epi32(0x80)), 24),+ _mm256_slli_epi32(low, 20));+ __m256 value = _mm256_mul_ps(_mm256_castsi256_ps(bits), _mm256_set1_ps(0x1p120f));+ __m256i nan = _mm256_cmpeq_epi32(low, _mm256_set1_epi32(0x7F));+ return _mm256_blendv_ps(value, _mm256_set1_ps(NAN), _mm256_castsi256_ps(nan));+}++/* Eight bf16 into eight floats: bf16 IS the top half of a float, so the whole+ * conversion is a shift. Exact for every encoding, NaNs and infinities+ * included, which is why there is no table to disagree with. */+static inline __m256 bf16_decode8(const uint16_t *p) {+ __m256i w = _mm256_cvtepu16_epi32(_mm_loadu_si128((const __m128i *)p));+ return _mm256_castsi256_ps(_mm256_slli_epi32(w, 16));+}+#endif+#define FP8_BLOCK 128static inline int64_t fp8_nblk(int n){ return ((int64_t)n + FP8_BLOCK - 1) / FP8_BLOCK; }diff --git a/c/tests/test_e4m3_vector.c b/c/tests/test_e4m3_vector.cnew file mode 100644index 00000000..6b7861d3--- /dev/null+++ b/c/tests/test_e4m3_vector.c@@ -0,0 +1,57 @@+/* The vector e4m3 decoder must agree with the table, for every byte.+ *+ * quant.h decodes e4m3 with a 256-entry lookup; e4m3_decode8 does it with two+ * shifts and a multiply, because a lookup per byte is what made V4.1's dense+ * attention 41% of a turn. The two must not diverge anywhere, and "anywhere"+ * is 256 values, so this checks all of them rather than a sample.+ *+ * Two cases are the reason this test exists rather than a comment:+ * - the subnormals, 0x01 to 0x07, which the trick produces by scaling a+ * subnormal float into the normal range. A build with flush-to-zero, or a+ * -ffast-math that implies it, turns them into zero and nothing else+ * notices: they are the smallest weights in the model.+ * - 0x7F and 0xFF, e4m3's NaNs, which the arithmetic would make into +-480.+ * A corrupt weight that becomes a plausible one is worse than one that+ * poisons the output, so those are blended back and asserted here.+ */+#include <stdio.h>+#include <stdint.h>+#include <math.h>+#include <string.h>++#include "../quant.h"++int main(void) {+#ifndef __AVX2__+ printf("SKIP: built without AVX2, the scalar table is the only decoder\n");+ return 0;+#else+ uint8_t bytes[256];+ for (int i = 0; i < 256; i++) bytes[i] = (uint8_t)i;+ int bad = 0, subnormals = 0, nans = 0;+ for (int base = 0; base < 256; base += 8) {+ float got[8];+ _mm256_storeu_ps(got, e4m3_decode8(bytes + base));+ for (int k = 0; k < 8; k++) {+ uint8_t b = bytes[base + k];+ float want = e4m3_decode(b);+ int both_nan = isnan(got[k]) && isnan(want);+ if (isnan(want)) nans++;+ if ((b & 0x7F) && !(b & 0x78)) subnormals++;+ if (!both_nan && memcmp(&got[k], &want, sizeof(float))) {+ if (bad < 8)+ printf(" FAIL 0x%02X: vector %.9g, table %.9g\n", b, got[k], want);+ bad++;+ }+ }+ }+ printf("e4m3: %d/256 bytes identical (%d subnormals, %d NaNs)\n",+ 256 - bad, subnormals, nans);+ if (subnormals != 14 || nans != 2) {+ printf("FAIL: the encoding is not what this test was written against\n");+ return 1;+ }+ printf("%s\n", bad ? "FAILED" : "PASS");+ return bad ? 1 : 0;+#endif+}diff --git a/c/tests/test_segment_conformance.c b/c/tests/test_segment_conformance.cindex 9ee6c616..7693dbbc 100644--- a/c/tests/test_segment_conformance.c+++ b/c/tests/test_segment_conformance.c@@ -3,17 +3,23 @@#include <stdio.h>#include <string.h>+/* Every family that must carry a Segment fixture. The count below is derived+ * from this list, not written a second time: a family added to one and not the+ * other is how this test last failed for a reason unrelated to what it checks. */+static const char *const kExpectedFamilies[] = {+ "glm", "glm53", "inkling", "kimi", "olmoe", "qwen36", "qwen38", "deepseek_v4",+ "deepseek_v41",+};+static int expected_family(const char *family_id) {- static const char *const expected[] = {- "glm", "glm53", "inkling", "kimi", "olmoe", "qwen36", "qwen38", "deepseek_v4",- };- for (size_t i = 0; i < sizeof(expected) / sizeof(expected[0]); i++)- if (strcmp(expected[i], family_id) == 0) return 1;+ for (size_t i = 0; i < sizeof(kExpectedFamilies) / sizeof(kExpectedFamilies[0]); i++)+ if (strcmp(kExpectedFamilies[i], family_id) == 0) return 1;return 0;}int main(void) {- const size_t required_families = 8;+ const size_t required_families =+ sizeof(kExpectedFamilies) / sizeof(kExpectedFamilies[0]);size_t count = coli_segment_conformance_fixture_count();if (count != required_families) {fprintf(stderr, "segment conformance requires %zu families, found %zu\n",