Run frontier MoE models on hardware you already own — pure C, zero deps, experts streamed from disk. Tiny engine, immense model. 🐦
C
olmoe: int8 container support (fixes SIGBUS), NEON int8 dot kernels (2.7x on ARM), macOS RSS fix, converter fallback (#187)
Louieec0aadf91afa
diff --git a/c/olmoe.c b/c/olmoe.cindex 882e1957..1c5ffaa8 100644--- a/c/olmoe.c+++ b/c/olmoe.c@@ -51,7 +51,11 @@ typedef struct {/* ---------- utility ---------- */static double now_s(void) { struct timespec t; clock_gettime(CLOCK_MONOTONIC, &t); return t.tv_sec + t.tv_nsec*1e-9; }-static double rss_gb(void) { struct rusage r; getrusage(RUSAGE_SELF, &r); return r.ru_maxrss / (1024.0*1024.0); }+#if defined(__APPLE__)+static double rss_gb(void) { struct rusage r; getrusage(RUSAGE_SELF, &r); return r.ru_maxrss / (1024.0*1024.0*1024.0); } /* macOS: byte */+#else+static double rss_gb(void) { struct rusage r; getrusage(RUSAGE_SELF, &r); return r.ru_maxrss / (1024.0*1024.0); } /* Linux: KB */+#endifstatic float *falloc(int64_t n) { float *p = malloc(n*sizeof(float)); if(!p){fprintf(stderr,"OOM %ld\n",(long)n);exit(1);} return p; }/* y[S,O] = x[S,I] @ W^T, W e' [O,I] row-major */@@ -69,8 +73,47 @@ static void matmul(float *y, const float *x, const float *W, int S, int I, int O}/* y[1,O] = x[1,I] @ W^T con W quantizzato: q[O,I] int8 + scala per riga.- * W[o,i] ~= q[o,i]*scale[o] -> y[o] = scale[o] * sum_i x[i]*q[o,i]. */+ * W[o,i] ~= q[o,i]*scale[o] -> y[o] = scale[o] * sum_i x[i]*q[o,i].+ * Su ARM: attivazione quantizzata Q8_0 (scala per blocco di 16) + dot int8+ * NEON (sdot dove c'e' dotprod) — stessa famiglia IDOT di glm.c, IDOT=0 per+ * la via scalare byte-esatta. Misurato 2.7x end-to-end su M5. */+#if defined(__ARM_NEON)+#include <arm_neon.h>+static inline int32_t dot_i8_16(const int8_t *a, const int8_t *b) {+ int32x4_t acc = vdupq_n_s32(0);+ int8x16_t va = vld1q_s8(a), vb = vld1q_s8(b);+#if defined(__ARM_FEATURE_DOTPROD)+ acc = vdotq_s32(acc, va, vb);+#else+ acc = vpadalq_s16(acc, vmull_s8(vget_low_s8(va), vget_low_s8(vb)));+ acc = vpadalq_s16(acc, vmull_s8(vget_high_s8(va), vget_high_s8(vb)));+#endif+ return vaddvq_s32(acc);+}+#endifstatic void matmul_q(float *y, const float *x, const int8_t *q, const float *scale, int I, int O) {+#if defined(__ARM_NEON)+ static int idot = -1;+ if (idot < 0) { const char *e = getenv("IDOT"); idot = !(e && *e == '0'); }+ if (idot && I % 16 == 0 && I <= 4096) {+ int nb = I / 16; int8_t xi[4096]; float xs[256];+ for (int b = 0; b < nb; b++) {+ const float *xb = x + b*16;+ float am = 0.f; for (int i = 0; i < 16; i++) { float a = fabsf(xb[i]); if (a > am) am = a; }+ float s = am/127.f; if (s < 1e-12f) s = 1e-12f;+ xs[b] = s; float inv = 1.f/s;+ for (int i = 0; i < 16; i++) xi[b*16+i] = (int8_t)lrintf(xb[i]*inv);+ }+ #pragma omp parallel for schedule(static)+ for (int o = 0; o < O; o++) {+ const int8_t *w = q + (int64_t)o * I;+ float acc = 0.f;+ for (int b = 0; b < nb; b++) acc += xs[b]*(float)dot_i8_16(xi+b*16, w+b*16);+ y[o] = acc * scale[o];+ }+ return;+ }+#endif#pragma omp parallel for schedule(static)for (int o = 0; o < O; o++) {const int8_t *w = q + (int64_t)o * I;@@ -171,8 +214,18 @@ static void model_init(Model *m, const char *snap, int cap, int bits) {m->dense_load_s = now_s() - t0;}-/* legge un weight dal disco (streaming) e lo quantizza in q[O,I]+scale[O] */+/* legge un weight dal disco (streaming) e lo quantizza in q[O,I]+scale[O].+ * Container pre-quantizzato (convert_olmoe.py: int8 + scale f32 in "name.qs"):+ * lettura raw diretta — meta' I/O e zero quantize_rows a runtime. Prima di+ * questa patch il container int8 causava SIGBUS (st_read_f32 su tensori I8). */static void load_expert_w(Model *m, const char *name, int8_t *q, float *scale, int O, int I, float *tmp) {+ st_tensor *t = st_find(&m->S, name);+ if (t && t->dtype == 3) { /* I8/U8: container colibri */+ char qs[300]; snprintf(qs, sizeof(qs), "%s.qs", name);+ st_read_raw(&m->S, name, q, 1);+ st_read_f32(&m->S, qs, scale, 1);+ return;+ }st_read_f32(&m->S, name, tmp, 1); /* pread + fadvise DONTNEED */quantize_rows(tmp, q, scale, O, I, m->quant_bits);}diff --git a/c/tools/convert_olmoe.py b/c/tools/convert_olmoe.pyindex 60d87444..dd458060 100644--- a/c/tools/convert_olmoe.py+++ b/c/tools/convert_olmoe.py@@ -55,9 +55,13 @@ def main():if args.repo:from huggingface_hub import snapshot_download+ from huggingface_hub.errors import LocalEntryNotFoundErrorprint(f"Downloading {args.repo}...")- src_dir = snapshot_download(args.repo, local_files_only=True, max_workers=4)- if not any(Path(src_dir).glob("*.safetensors")):+ try:+ src_dir = snapshot_download(args.repo, local_files_only=True, max_workers=4)+ except LocalEntryNotFoundError:+ src_dir = None+ if src_dir is None or not any(Path(src_dir).glob("*.safetensors")):print("Downloading safetensors...")src_dir = snapshot_download(args.repo, max_workers=4)else: