Run frontier MoE models on hardware you already own — pure C, zero deps, experts streamed from disk. Tiny engine, immense model. 🐦
C
olmoe: SSE 4.1 SSE4.1 tier canary (reference for future engine ports)
Jaapjan Tinbergen179a4ccef8f2
diff --git a/c/olmoe.c b/c/olmoe.cindex 45598832..8dd5f92e 100644--- a/c/olmoe.c+++ b/c/olmoe.c@@ -265,6 +265,29 @@ static inline int32_t dot_i8_16(const int8_t *a, const int8_t *b) {return _mm_cvtsi128_si32(sum32);}#define HAVE_FAST_DOT_I8 1+#elif defined(__SSE4_1__)+#include <immintrin.h>+#include "sse41_kernels.h"+/* Sandy Bridge-EP path: AVX 1.0 only, no FMA, no AVX-2.+ * 16 int8 dot via two 8-wide SSE2 sign-extend + SSE4.1 madd pairs.+ * Bit-for-bit identical to the AVX2 version above (just 2x 128-bit ops+ * instead of 1x 256-bit op). NO FMA here -- this branch targets Sandy Bridge+ * which has no FMA -- so use explicit mul+add for the inner accumulation. */+static inline int32_t dot_i8_16(const int8_t *a, const int8_t *b) {+ __m128i va_lo = _mm_cvtepi8_epi16(_mm_loadu_si128((const __m128i*)a)); /* lower 8 int8 -> 8 int16 */+ __m128i vb_lo = _mm_cvtepi8_epi16(_mm_loadu_si128((const __m128i*)b));+ __m128i va_hi = _mm_cvtepi8_epi16(_mm_loadu_si128((const __m128i*)(a + 8))); /* upper 8 int8 -> 8 int16 */+ __m128i vb_hi = _mm_cvtepi8_epi16(_mm_loadu_si128((const __m128i*)(b + 8)));+ __m128i p_lo = _mm_madd_epi16(va_lo, vb_lo); /* 4 x int32 from 8 int16 pairs */+ __m128i p_hi = _mm_madd_epi16(va_hi, vb_hi); /* 4 x int32 from 8 int16 pairs */+ __m128i sum = _mm_add_epi32(p_lo, p_hi);+ /* horizontal reduce 4 x int32 -> 1 x int32 */+ __m128i hi64 = _mm_unpackhi_epi64(sum, sum);+ __m128i sum64 = _mm_add_epi32(sum, hi64);+ __m128i hi32 = _mm_shuffle_epi32(sum64, _MM_SHUFFLE(2, 3, 0, 1));+ return _mm_cvtsi128_si32(_mm_add_epi32(sum64, hi32));+}+#define HAVE_FAST_DOT_I8 1#endif/* Test-only hook, compiled out of the shipping binary.*diff --git a/c/sse41_kernels.h b/c/sse41_kernels.hnew file mode 100644index 00000000..6c31dcce--- /dev/null+++ b/c/sse41_kernels.h@@ -0,0 +1,62 @@+#ifndef COLIBRI_SSE41_KERNELS_H+#define COLIBRI_SSE41_KERNELS_H+/*+ * sse41_kernels.h — shared SSE 4.1 primitives for Colibri engines.+ *+ * The engines (c/deepseek_v4.c, c/colibri.c, c/kimi_k3.c, c/olmoe.c, c/inkling.c)+ * have an `#if defined(__AVX2__)` dispatch for fast paths and fall through to scalar+ * on pre-Haswell hardware (Sandy Bridge: AVX 1.0, no FMA, no AVX-2). This header+ * provides the missing middle tier: 128-bit SIMD primitives, FMA-free.+ *+ * Why a separate header (not just inline in each .c):+ * - 109 AVX2 sites total across 5 engines need patching. Copy-pasting the+ * 128-bit intrinsics 109 times is a typo factory. A single macro definition+ * is the difference between correct and wrong-on-100-sites.+ * - The single most critical shared piece is the FMA-emulation macro: on+ * Sandy Bridge, _mm_mul_ps + _mm_add_ps has double rounding vs. hardware+ * FMA's single rounding, so the output is NOT bit-identical to AVX2 (1-2 ULP+ * difference). A typo in a single copy is a silent correctness bug.+ *+ * This is the minimum needed for the SSE 4.1 fallback. More primitives can be+ * added as additional engines are patched.+ */+#if defined(__SSE2__)++/*+ * COLIBRI_FMA: emulate FMA on non-FMA hardware.+ *+ * On FMA hardware: maps to _mm_fmadd_ps (single rounding, 1 instruction).+ * On Sandy Bridge (no FMA): separate mul+add, double rounding, 2 instructions.+ *+ * On Sandy Bridge this is NOT bit-identical to AVX2/FMA output — typically+ * within 1-2 ULP. Test tolerance must accommodate this.+ */+#if defined(__FMA__)+# define COLIBRI_FMA(a, b, c) _mm_fmadd_ps((a), (b), (c))+#else+# define COLIBRI_FMA(a, b, c) _mm_add_ps(_mm_mul_ps((a), (b)), (c))+#endif++/*+ * SSE 4.1 (or lower) load/store helpers. Sandy Bridge has these natively.+ * _mm_load_ps is aligned; _mm_loadu_ps is unaligned. For 128-bit (16-byte)+ * data, aligned loads are faster but UB on misaligned pointers. Default to+ * unaligned: buffers from malloc / numa_slab_bind have no 16-byte guarantee.+ * Aligned loads can be added as a profiled follow-up.+ */+static inline __m128 colibri_sse41_loadu_ps(const float *p) { return _mm_loadu_ps(p); }+static inline void colibri_sse41_storeu_ps(float *p, __m128 v) { _mm_storeu_ps(p, v); }++/*+ * Min/max (SSE 4.1 native). Identical to AVX2, just narrower width.+ */+static inline __m128 colibri_sse41_min_ps(__m128 a, __m128 b) { return _mm_min_ps(a, b); }+static inline __m128 colibri_sse41_max_ps(__m128 a, __m128 b) { return _mm_max_ps(a, b); }++/*+ * Prefetch (SSE 1+, always available). Identical to AVX2/FMA path.+ */+static inline void colibri_sse41_prefetch(const void *p) { _mm_prefetch(p, _MM_HINT_T0); }++#endif /* __SSE2__ */+#endif /* COLIBRI_SSE41_KERNELS_H */