Skip to content

Run frontier MoE models on hardware you already own — pure C, zero deps, experts streamed from disk. Tiny engine, immense model. 🐦

C

olmoe: SSE 4.1 SSE4.1 tier canary (reference for future engine ports)

Jaapjan Tinbergen179a4ccef8f2
diff --git a/c/olmoe.c b/c/olmoe.c
index 45598832..8dd5f92e 100644
--- a/c/olmoe.c
+++ b/c/olmoe.c
@@ -265,6 +265,29 @@ static inline int32_t dot_i8_16(const int8_t *a, const int8_t *b) {
return _mm_cvtsi128_si32(sum32);
}
#define HAVE_FAST_DOT_I8 1
+#elif defined(__SSE4_1__)
+#include <immintrin.h>
+#include "sse41_kernels.h"
+/* Sandy Bridge-EP path: AVX 1.0 only, no FMA, no AVX-2.
+ * 16 int8 dot via two 8-wide SSE2 sign-extend + SSE4.1 madd pairs.
+ * Bit-for-bit identical to the AVX2 version above (just 2x 128-bit ops
+ * instead of 1x 256-bit op). NO FMA here -- this branch targets Sandy Bridge
+ * which has no FMA -- so use explicit mul+add for the inner accumulation. */
+static inline int32_t dot_i8_16(const int8_t *a, const int8_t *b) {
+ __m128i va_lo = _mm_cvtepi8_epi16(_mm_loadu_si128((const __m128i*)a)); /* lower 8 int8 -> 8 int16 */
+ __m128i vb_lo = _mm_cvtepi8_epi16(_mm_loadu_si128((const __m128i*)b));
+ __m128i va_hi = _mm_cvtepi8_epi16(_mm_loadu_si128((const __m128i*)(a + 8))); /* upper 8 int8 -> 8 int16 */
+ __m128i vb_hi = _mm_cvtepi8_epi16(_mm_loadu_si128((const __m128i*)(b + 8)));
+ __m128i p_lo = _mm_madd_epi16(va_lo, vb_lo); /* 4 x int32 from 8 int16 pairs */
+ __m128i p_hi = _mm_madd_epi16(va_hi, vb_hi); /* 4 x int32 from 8 int16 pairs */
+ __m128i sum = _mm_add_epi32(p_lo, p_hi);
+ /* horizontal reduce 4 x int32 -> 1 x int32 */
+ __m128i hi64 = _mm_unpackhi_epi64(sum, sum);
+ __m128i sum64 = _mm_add_epi32(sum, hi64);
+ __m128i hi32 = _mm_shuffle_epi32(sum64, _MM_SHUFFLE(2, 3, 0, 1));
+ return _mm_cvtsi128_si32(_mm_add_epi32(sum64, hi32));
+}
+#define HAVE_FAST_DOT_I8 1
#endif
/* Test-only hook, compiled out of the shipping binary.
*
diff --git a/c/sse41_kernels.h b/c/sse41_kernels.h
new file mode 100644
index 00000000..6c31dcce
--- /dev/null
+++ b/c/sse41_kernels.h
@@ -0,0 +1,62 @@
+#ifndef COLIBRI_SSE41_KERNELS_H
+#define COLIBRI_SSE41_KERNELS_H
+/*
+ * sse41_kernels.h — shared SSE 4.1 primitives for Colibri engines.
+ *
+ * The engines (c/deepseek_v4.c, c/colibri.c, c/kimi_k3.c, c/olmoe.c, c/inkling.c)
+ * have an `#if defined(__AVX2__)` dispatch for fast paths and fall through to scalar
+ * on pre-Haswell hardware (Sandy Bridge: AVX 1.0, no FMA, no AVX-2). This header
+ * provides the missing middle tier: 128-bit SIMD primitives, FMA-free.
+ *
+ * Why a separate header (not just inline in each .c):
+ * - 109 AVX2 sites total across 5 engines need patching. Copy-pasting the
+ * 128-bit intrinsics 109 times is a typo factory. A single macro definition
+ * is the difference between correct and wrong-on-100-sites.
+ * - The single most critical shared piece is the FMA-emulation macro: on
+ * Sandy Bridge, _mm_mul_ps + _mm_add_ps has double rounding vs. hardware
+ * FMA's single rounding, so the output is NOT bit-identical to AVX2 (1-2 ULP
+ * difference). A typo in a single copy is a silent correctness bug.
+ *
+ * This is the minimum needed for the SSE 4.1 fallback. More primitives can be
+ * added as additional engines are patched.
+ */
+#if defined(__SSE2__)
+
+/*
+ * COLIBRI_FMA: emulate FMA on non-FMA hardware.
+ *
+ * On FMA hardware: maps to _mm_fmadd_ps (single rounding, 1 instruction).
+ * On Sandy Bridge (no FMA): separate mul+add, double rounding, 2 instructions.
+ *
+ * On Sandy Bridge this is NOT bit-identical to AVX2/FMA output — typically
+ * within 1-2 ULP. Test tolerance must accommodate this.
+ */
+#if defined(__FMA__)
+# define COLIBRI_FMA(a, b, c) _mm_fmadd_ps((a), (b), (c))
+#else
+# define COLIBRI_FMA(a, b, c) _mm_add_ps(_mm_mul_ps((a), (b)), (c))
+#endif
+
+/*
+ * SSE 4.1 (or lower) load/store helpers. Sandy Bridge has these natively.
+ * _mm_load_ps is aligned; _mm_loadu_ps is unaligned. For 128-bit (16-byte)
+ * data, aligned loads are faster but UB on misaligned pointers. Default to
+ * unaligned: buffers from malloc / numa_slab_bind have no 16-byte guarantee.
+ * Aligned loads can be added as a profiled follow-up.
+ */
+static inline __m128 colibri_sse41_loadu_ps(const float *p) { return _mm_loadu_ps(p); }
+static inline void colibri_sse41_storeu_ps(float *p, __m128 v) { _mm_storeu_ps(p, v); }
+
+/*
+ * Min/max (SSE 4.1 native). Identical to AVX2, just narrower width.
+ */
+static inline __m128 colibri_sse41_min_ps(__m128 a, __m128 b) { return _mm_min_ps(a, b); }
+static inline __m128 colibri_sse41_max_ps(__m128 a, __m128 b) { return _mm_max_ps(a, b); }
+
+/*
+ * Prefetch (SSE 1+, always available). Identical to AVX2/FMA path.
+ */
+static inline void colibri_sse41_prefetch(const void *p) { _mm_prefetch(p, _MM_HINT_T0); }
+
+#endif /* __SSE2__ */
+#endif /* COLIBRI_SSE41_KERNELS_H */