Skip to content

Run frontier MoE models on hardware you already own — pure C, zero deps, experts streamed from disk. Tiny engine, immense model. 🐦

C

perf(deepseek-v41): the FP8 and BF16 inner loops go through AVX2

JustVugg39ca742ec8be
diff --git a/c/Makefile b/c/Makefile
index 86193f9d..a76c5e8d 100644
--- a/c/Makefile
+++ b/c/Makefile
@@ -1358,6 +1358,9 @@ tests/test_kv_disk$(EXE): tests/test_kv_disk.c colibri.c st.h json.h tok.h tok_u
tests/test_e8_kernel$(EXE): tests/test_e8_kernel.c quant.h
$(CC) $(CFLAGS) $< -o $@ $(LDFLAGS)
+tests/test_e4m3_vector$(EXE): tests/test_e4m3_vector.c quant.h
+ $(CC) $(CFLAGS) $< -o $@ $(LDFLAGS)
+
tests/test_rans$(EXE): tests/test_rans.c rans.h compat.h
$(CC) $(CFLAGS) $< -o $@ $(LDFLAGS)
diff --git a/c/deepseek_v41.c b/c/deepseek_v41.c
index 0b5b3b38..7f5c54fa 100644
--- a/c/deepseek_v41.c
+++ b/c/deepseek_v41.c
@@ -58,6 +58,9 @@
#include "st.h"
#include "quant.h"
#include <pthread.h> /* ehit_mark publishes the lazy HITS table under a lock */
+#if defined(__AVX2__)
+#include <immintrin.h>
+#endif
#include "hyper_connections.h"
#include "tok.h"
#include "serve_codec.h"
@@ -321,7 +324,13 @@ static inline float ue8m0(uint8_t byte) {
return value.f;
}
-/* y[O] = W [O, I] x[I], W in e4m3 with one ue8m0 scale per 32x32 tile. */
+/* y[O] = W [O, I] x[I], W in e4m3 with one ue8m0 scale per 32x32 tile.
+ *
+ * The dense trunk's matvec, and the reason it is worth vectorising: V4.1's
+ * attention projects 64 heads of 512, so wq_b alone is 32768 x 1536 per layer
+ * and forty layers of it are several GFLOP per token. Measured on the released
+ * checkpoint, the attention block was 41% of a turn's wall clock -- more than
+ * the expert reads from disk. A scalar byte-at-a-time decode was most of it. */
static void mv8(float *y, const W8 *w, const float *x) {
int I = w->I, tiles_i = (I + FP8_TILE - 1) / FP8_TILE;
#pragma omp parallel for schedule(static)
@@ -333,6 +342,15 @@ static void mv8(float *y, const W8 *w, const float *x) {
int width = I - base < FP8_TILE ? I - base : FP8_TILE;
float tile = ue8m0(scale[base / FP8_TILE]);
float part = 0.0f;
+#if defined(__AVX2__)
+ if (width == FP8_TILE) {
+ __m256 acc = _mm256_setzero_ps();
+ for (int i = 0; i < FP8_TILE; i += 8)
+ acc = _mm256_fmadd_ps(e4m3_decode8(row + base + i),
+ _mm256_loadu_ps(x + base + i), acc);
+ part = hsum256(acc);
+ } else
+#endif
for (int i = 0; i < width; i++) part += e4m3_decode(row[base + i]) * x[base + i];
sum += part * tile;
}
@@ -346,7 +364,16 @@ static void mvb(float *y, const WB *w, const float *x) {
for (int o = 0; o < w->O; o++) {
const uint16_t *row = w->w + (size_t)o * I;
float sum = 0.0f;
- for (int i = 0; i < I; i++) sum += bf16_to_f32(row[i]) * x[i];
+ int i = 0;
+#if defined(__AVX2__)
+ /* The lm head is [vocab, dim] in bf16: 129280 x 5120 is 662 million
+ * multiply-adds for one token, and it runs once per position. */
+ __m256 acc = _mm256_setzero_ps();
+ for (; i + 8 <= I; i += 8)
+ acc = _mm256_fmadd_ps(bf16_decode8(row + i), _mm256_loadu_ps(x + i), acc);
+ sum = hsum256(acc);
+#endif
+ for (; i < I; i++) sum += bf16_to_f32(row[i]) * x[i];
y[o] = sum;
}
}
@@ -1329,6 +1356,15 @@ static void attn_project_out(Model *m, Layer *l, const float *heads, float *out)
int width = per_group - base < FP8_TILE ? per_group - base : FP8_TILE;
float tile = ue8m0(scale_row[base / FP8_TILE]);
float part = 0.0f;
+#if defined(__AVX2__)
+ if (width == FP8_TILE) {
+ __m256 acc = _mm256_setzero_ps();
+ for (int i = 0; i < FP8_TILE; i += 8)
+ acc = _mm256_fmadd_ps(e4m3_decode8(w + base + i),
+ _mm256_loadu_ps(xg + base + i), acc);
+ part = hsum256(acc);
+ } else
+#endif
for (int i = 0; i < width; i++) part += e4m3_decode(w[base + i]) * xg[base + i];
sum += part * tile;
}
@@ -2873,6 +2909,23 @@ static void serve_loop(Model *m, Tok *tokenizer, const char *snap) {
}
}
free(batch); free(confidence);
+ if (m->engram.active) {
+ /* What the n-gram memory actually cost this turn. Rows are 264
+ * bytes and the traffic is Zipfian, so the useful question is not
+ * how many rows were read but how many of them the cache already
+ * had -- and, on a turn short enough that capacity cannot bind,
+ * what is left is pure repetition within the text. */
+ for (int t = 0; t < m->engram.n_layers; t++) {
+ EngramTable *tab = &m->engram.table[t];
+ uint64_t total = tab->hits + tab->misses;
+ fprintf(stderr, "[v41] engram table %d (layer %d): %llu lookups, "
+ "%.0f%% cache hits, %llu rows read (%.1f MB)\n",
+ t, m->engram.layer_of[t], (unsigned long long)total,
+ total ? 100.0 * (double)tab->hits / (double)total : 0.0,
+ (unsigned long long)tab->misses,
+ tab->misses * (double)(m->engram.head_dim + m->engram.head_dim / 32) / 1e6);
+ }
+ }
if (m->spec.active && m->spec.proposed > proposed0)
fprintf(stderr, "[v41] DSpark: %llu of %llu drafts accepted this turn\n",
(unsigned long long)(m->spec.accepted - accepted0),
diff --git a/c/quant.h b/c/quant.h
index 31ac13a1..f53ad436 100644
--- a/c/quant.h
+++ b/c/quant.h
@@ -511,6 +511,41 @@ static const float E4M3_LUT[256] = {
};
static inline float e4m3_decode(uint8_t b){ return E4M3_LUT[b]; }
+#ifdef __AVX2__
+/* Eight e4m3 bytes to eight floats without touching a lookup table.
+ *
+ * A float wants sign, exponent, mantissa in that order, and e4m3 already has
+ * them in that order: place the byte's seven low bits at bit 20 and its sign at
+ * bit 31, and the result is the right number with the wrong exponent bias,
+ * 127 where e4m3 means 7. One multiply by 2^120 fixes it, and it fixes the
+ * subnormals too, because scaling a subnormal by a power of two into the normal
+ * range is exact.
+ *
+ * Verified byte for byte against quant.h's E4M3_LUT: 254 of the 256 encodings
+ * come out identical. The two that do not are 0x7F and 0xFF, e4m3's NaNs, which
+ * this would turn into +-480 -- a corrupt weight quietly becoming a plausible
+ * one. They are blended back, because a NaN that propagates is how you find out.
+ */
+static inline __m256 e4m3_decode8(const uint8_t *p) {
+ __m256i b = _mm256_cvtepu8_epi32(_mm_loadl_epi64((const __m128i *)p));
+ __m256i low = _mm256_and_si256(b, _mm256_set1_epi32(0x7F));
+ __m256i bits = _mm256_or_si256(
+ _mm256_slli_epi32(_mm256_and_si256(b, _mm256_set1_epi32(0x80)), 24),
+ _mm256_slli_epi32(low, 20));
+ __m256 value = _mm256_mul_ps(_mm256_castsi256_ps(bits), _mm256_set1_ps(0x1p120f));
+ __m256i nan = _mm256_cmpeq_epi32(low, _mm256_set1_epi32(0x7F));
+ return _mm256_blendv_ps(value, _mm256_set1_ps(NAN), _mm256_castsi256_ps(nan));
+}
+
+/* Eight bf16 into eight floats: bf16 IS the top half of a float, so the whole
+ * conversion is a shift. Exact for every encoding, NaNs and infinities
+ * included, which is why there is no table to disagree with. */
+static inline __m256 bf16_decode8(const uint16_t *p) {
+ __m256i w = _mm256_cvtepu16_epi32(_mm_loadu_si128((const __m128i *)p));
+ return _mm256_castsi256_ps(_mm256_slli_epi32(w, 16));
+}
+#endif
+
#define FP8_BLOCK 128
static inline int64_t fp8_nblk(int n){ return ((int64_t)n + FP8_BLOCK - 1) / FP8_BLOCK; }
diff --git a/c/tests/test_e4m3_vector.c b/c/tests/test_e4m3_vector.c
new file mode 100644
index 00000000..6b7861d3
--- /dev/null
+++ b/c/tests/test_e4m3_vector.c
@@ -0,0 +1,57 @@
+/* The vector e4m3 decoder must agree with the table, for every byte.
+ *
+ * quant.h decodes e4m3 with a 256-entry lookup; e4m3_decode8 does it with two
+ * shifts and a multiply, because a lookup per byte is what made V4.1's dense
+ * attention 41% of a turn. The two must not diverge anywhere, and "anywhere"
+ * is 256 values, so this checks all of them rather than a sample.
+ *
+ * Two cases are the reason this test exists rather than a comment:
+ * - the subnormals, 0x01 to 0x07, which the trick produces by scaling a
+ * subnormal float into the normal range. A build with flush-to-zero, or a
+ * -ffast-math that implies it, turns them into zero and nothing else
+ * notices: they are the smallest weights in the model.
+ * - 0x7F and 0xFF, e4m3's NaNs, which the arithmetic would make into +-480.
+ * A corrupt weight that becomes a plausible one is worse than one that
+ * poisons the output, so those are blended back and asserted here.
+ */
+#include <stdio.h>
+#include <stdint.h>
+#include <math.h>
+#include <string.h>
+
+#include "../quant.h"
+
+int main(void) {
+#ifndef __AVX2__
+ printf("SKIP: built without AVX2, the scalar table is the only decoder\n");
+ return 0;
+#else
+ uint8_t bytes[256];
+ for (int i = 0; i < 256; i++) bytes[i] = (uint8_t)i;
+ int bad = 0, subnormals = 0, nans = 0;
+ for (int base = 0; base < 256; base += 8) {
+ float got[8];
+ _mm256_storeu_ps(got, e4m3_decode8(bytes + base));
+ for (int k = 0; k < 8; k++) {
+ uint8_t b = bytes[base + k];
+ float want = e4m3_decode(b);
+ int both_nan = isnan(got[k]) && isnan(want);
+ if (isnan(want)) nans++;
+ if ((b & 0x7F) && !(b & 0x78)) subnormals++;
+ if (!both_nan && memcmp(&got[k], &want, sizeof(float))) {
+ if (bad < 8)
+ printf(" FAIL 0x%02X: vector %.9g, table %.9g\n", b, got[k], want);
+ bad++;
+ }
+ }
+ }
+ printf("e4m3: %d/256 bytes identical (%d subnormals, %d NaNs)\n",
+ 256 - bad, subnormals, nans);
+ if (subnormals != 14 || nans != 2) {
+ printf("FAIL: the encoding is not what this test was written against\n");
+ return 1;
+ }
+ printf("%s\n", bad ? "FAILED" : "PASS");
+ return bad ? 1 : 0;
+#endif
+}
diff --git a/c/tests/test_segment_conformance.c b/c/tests/test_segment_conformance.c
index 9ee6c616..7693dbbc 100644
--- a/c/tests/test_segment_conformance.c
+++ b/c/tests/test_segment_conformance.c
@@ -3,17 +3,23 @@
#include <stdio.h>
#include <string.h>
+/* Every family that must carry a Segment fixture. The count below is derived
+ * from this list, not written a second time: a family added to one and not the
+ * other is how this test last failed for a reason unrelated to what it checks. */
+static const char *const kExpectedFamilies[] = {
+ "glm", "glm53", "inkling", "kimi", "olmoe", "qwen36", "qwen38", "deepseek_v4",
+ "deepseek_v41",
+};
+
static int expected_family(const char *family_id) {
- static const char *const expected[] = {
- "glm", "glm53", "inkling", "kimi", "olmoe", "qwen36", "qwen38", "deepseek_v4",
- };
- for (size_t i = 0; i < sizeof(expected) / sizeof(expected[0]); i++)
- if (strcmp(expected[i], family_id) == 0) return 1;
+ for (size_t i = 0; i < sizeof(kExpectedFamilies) / sizeof(kExpectedFamilies[0]); i++)
+ if (strcmp(kExpectedFamilies[i], family_id) == 0) return 1;
return 0;
}
int main(void) {
- const size_t required_families = 8;
+ const size_t required_families =
+ sizeof(kExpectedFamilies) / sizeof(kExpectedFamilies[0]);
size_t count = coli_segment_conformance_fixture_count();
if (count != required_families) {
fprintf(stderr, "segment conformance requires %zu families, found %zu\n",