Run frontier MoE models on hardware you already own — pure C, zero deps, experts streamed from disk. Tiny engine, immense model. 🐦
C
tier: do not inherit the OpenMP master's CPU mask for the uploader and CUDA threads
kreuzzelg792b6f2de62d
diff --git a/c/qwen36_tier.c b/c/qwen36_tier.cindex 81299801..2e610715 100644--- a/c/qwen36_tier.c+++ b/c/qwen36_tier.c@@ -4,6 +4,10 @@#include <stdlib.h>#include <string.h>#include <pthread.h>+#ifdef __linux__+#include <unistd.h>+#include <sys/syscall.h>+#endif#include "qwen36_tier.h"#include "backend_cuda.h"#include "tier.h"@@ -82,8 +86,56 @@ static void stage(uint8_t *dw, float *dsc,memcpy(dsc+2*G.sc_gu, ds, G.sc_d *sizeof(float));}+/* Thread affinity around the tier's own threads (Linux).+ *+ * With OMP_PROC_BIND set, libgomp binds the initial thread to place 0 before+ * main() runs, and a pthread inherits the CPU mask of the thread that creates+ * it. The uploader thread and the CUDA runtime's own threads were therefore+ * jailed on the OpenMP master's core: every staging copy and every driver+ * call competed with the master thread's share of each expert matmul, and+ * the whole team waited for it. Measured on Qwen3.8 (12 threads, one card):+ * the CPU time per remaining expert rose 64 % while the tier was on, eating+ * the whole gain of computing 45-59 % of the experts on the GPU. So the tier+ * widens the calling thread's mask to every online CPU while it creates its+ * thread and initializes CUDA, and restores the caller's mask afterwards.+ * Raw syscalls, no _GNU_SOURCE: this file is also #included by tests after+ * the engine's own headers. */+#ifdef __linux__+#define QT_AFF_WORDS 64 /* 4096 CPUs */+typedef struct { unsigned long w[QT_AFF_WORDS]; int len; } qt_affmask;+static int qt_aff_get(qt_affmask *m){+ long r=syscall(SYS_sched_getaffinity,0,sizeof m->w,m->w);+ if(r<=0) return 0;+ m->len=(int)r; return 1;+}+static void qt_aff_widen(const qt_affmask *saved){+ if(!saved->len) return;+ long n=sysconf(_SC_NPROCESSORS_ONLN);+ if(n<=1) return;+ qt_affmask all; memset(&all,0,sizeof all);+ for(long i=0;i<n && i<(long)(8*sizeof all.w);i++) all.w[i/(8*sizeof(unsigned long))] |= 1ul<<(i%(8*sizeof(unsigned long)));+ syscall(SYS_sched_setaffinity,0,(size_t)saved->len,all.w);+}+static void qt_aff_restore(const qt_affmask *saved){+ if(saved->len) syscall(SYS_sched_setaffinity,0,(size_t)saved->len,saved->w);+}+static int qt_aff_count_self(void){+ qt_affmask m; if(!qt_aff_get(&m)) return 0;+ int c=0; for(int i=0;i<m.len/(int)sizeof(unsigned long);i++) c+=__builtin_popcountl(m.w[i]);+ return c;+}+#else+typedef struct { int len; } qt_affmask;+static int qt_aff_get(qt_affmask *m){ m->len=0; return 0; }+static void qt_aff_widen(const qt_affmask *m){ (void)m; }+static void qt_aff_restore(const qt_affmask *m){ (void)m; }+static int qt_aff_count_self(void){ return 0; }+#endif+static int G_uploader_cpus; /* CPUs the uploader thread may run on (0 = unknown) */+static void *uploader(void *arg){(void)arg;+ G_uploader_cpus=qt_aff_count_self();for(;;){pthread_mutex_lock(&G.mx);while(G.qn==0 && !G.th_stop) pthread_cond_wait(&G.cv,&G.mx);@@ -459,7 +511,10 @@ int qt_init(int nl, int ne, int D, int Ih, int cap, int topk, int expert_gs,}}if(G.ndev<1){ fprintf(stderr,"[qtier] no visible CUDA devices -> CPU path\n"); return 0; }- if(!coli_cuda_init(G.dev,G.ndev)){ fprintf(stderr,"[qtier] coli_cuda_init failed -> CPU path\n"); return 0; }+ qt_affmask aff; qt_aff_get(&aff); qt_aff_widen(&aff); /* CUDA's threads are born here */+ int cuda_ok_=coli_cuda_init(G.dev,G.ndev);+ qt_aff_restore(&aff);+ if(!cuda_ok_){ fprintf(stderr,"[qtier] coli_cuda_init failed -> CPU path\n"); return 0; }int have=coli_cuda_device_count();if(have<G.ndev){ G.ndev=have; }if(G.ndev<1){ fprintf(stderr,"[qtier] no CUDA devices -> CPU path\n"); return 0; }@@ -623,7 +678,10 @@ int qt_init(int nl, int ne, int D, int Ih, int cap, int topk, int expert_gs,G.is_x=malloc(G.is_x_floats*sizeof(float));if(!G.is_x) return 0;pthread_mutex_init(&G.mx,NULL); pthread_cond_init(&G.cv,NULL); pthread_cond_init(&G.cv_take,NULL);- if(pthread_create(&G.th,NULL,uploader,NULL)!=0) return 0;+ qt_aff_get(&aff); qt_aff_widen(&aff); /* the uploader inherits this mask */+ int th_ok=pthread_create(&G.th,NULL,uploader,NULL)==0;+ qt_aff_restore(&aff);+ if(!th_ok) return 0;G.on=1;fprintf(stderr,"[qtier] CUDA VRAM expert tier active: %d device(s), %.2f MB/expert\n",G.ndev, G.exp_bytes/1048576.0);diff --git a/c/tests/test_qwen36_tier_fp8.c b/c/tests/test_qwen36_tier_fp8.cindex 5abc8723..6b7689cd 100644--- a/c/tests/test_qwen36_tier_fp8.c+++ b/c/tests/test_qwen36_tier_fp8.c@@ -65,6 +65,11 @@ int main(void) {/* ---- 1. the mode starts with cap < n_experts, publishes the LUT, sizes fmt 8 ---- */printf(" 1. init\n");+#ifdef __linux__+ /* what libgomp does with OMP_PROC_BIND: the initial thread sits on one CPU */+ unsigned long one[QT_AFF_WORDS]; memset(one, 0, sizeof one); one[0] = 1ul;+ int bound = syscall(SYS_sched_setaffinity, 0, sizeof one, one) == 0;+#endif/* allowance for exactly 3 experts on the one device */size_t exp_bytes = 3 * dev_alloc_footprint((size_t)D * IH) + 3 * dev_alloc_footprint(NSC * sizeof(float));char gb[64]; snprintf(gb, sizeof gb, "%.15f", (double)(3 * exp_bytes + exp_bytes / 2) / 1073741824.0);@@ -75,6 +80,13 @@ int main(void) {check(fake_lut_published, "e4m3 LUT published to the backend before any upload");check(G.exp_bytes == exp_bytes, "exp_bytes = three fmt-8 matrices + three scale tables at cudaMalloc granularity");check(G.sc_gu == NSC && G.sc_d == NSC, "one scale per 128x128 block per matrix");+#ifdef __linux__+ if (bound && sysconf(_SC_NPROCESSORS_ONLN) > 1) {+ for (int i = 0; i < 200 && !G_uploader_cpus; i++) { struct timespec ts = {0, 1000000}; nanosleep(&ts, NULL); }+ check(G_uploader_cpus > 1, "uploader thread is not jailed on the caller's CPU");+ check(qt_aff_count_self() == 1, "the caller's own mask is restored after init");+ }+#endif/* ---- 2. a note uploads fmt 8, bytes intact, scales in order, pointers dropped ---- */printf(" 2. note -> upload\n");