Run frontier MoE models on hardware you already own — pure C, zero deps, experts streamed from disk. Tiny engine, immense model. 🐦
C
olmoe: add an AVX2 int8 dot-product path to matmul_q (x86 had none)
BE2c4e9de9e09a
diff --git a/c/olmoe.c b/c/olmoe.cindex d9460d40..ffeb803d 100644--- a/c/olmoe.c+++ b/c/olmoe.c@@ -149,9 +149,30 @@ static inline int32_t dot_i8_16(const int8_t *a, const int8_t *b) {#endifreturn vaddvq_s32(acc);}+#define HAVE_FAST_DOT_I8 1+#elif defined(__AVX2__)+#include <immintrin.h>+/* x86 counterpart of the NEON path above (was scalar-only here before —+ * the only fast path was ARM, so x86 boxes silently used the scalar+ * fallback even when AVX2 was available).+ * Sign-extend both int8 vectors to int16 (exact, no precision loss) then+ * madd+horizontal-sum in int32: pure integer arithmetic, so this is+ * bit-for-bit identical to the scalar dot product, just vectorized. */+static inline int32_t dot_i8_16(const int8_t *a, const int8_t *b) {+ __m256i va16 = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*)a));+ __m256i vb16 = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*)b));+ __m256i prod = _mm256_madd_epi16(va16, vb16); /* 8 x int32, adjacent pairs summed */+ __m128i sum128 = _mm_add_epi32(_mm256_castsi256_si128(prod), _mm256_extracti128_si256(prod, 1));+ __m128i hi64 = _mm_unpackhi_epi64(sum128, sum128);+ __m128i sum64 = _mm_add_epi32(sum128, hi64);+ __m128i hi32 = _mm_shuffle_epi32(sum64, _MM_SHUFFLE(2, 3, 0, 1));+ __m128i sum32 = _mm_add_epi32(sum64, hi32);+ return _mm_cvtsi128_si32(sum32);+}+#define HAVE_FAST_DOT_I8 1#endifstatic void matmul_q(float *y, const float *x, const int8_t *q, const float *scale, int I, int O) {-#if defined(__ARM_NEON)+#if defined(HAVE_FAST_DOT_I8)static int idot = -1;if (idot < 0) { const char *e = getenv("IDOT"); idot = !(e && *e == '0'); }if (idot && I % 16 == 0 && I <= 4096) {