Skip to content

Run frontier MoE models on hardware you already own — pure C, zero deps, experts streamed from disk. Tiny engine, immense model. 🐦

C

tier: do not inherit the OpenMP master's CPU mask for the uploader and CUDA threads

kreuzzelg792b6f2de62d
diff --git a/c/qwen36_tier.c b/c/qwen36_tier.c
index 81299801..2e610715 100644
--- a/c/qwen36_tier.c
+++ b/c/qwen36_tier.c
@@ -4,6 +4,10 @@
#include <stdlib.h>
#include <string.h>
#include <pthread.h>
+#ifdef __linux__
+#include <unistd.h>
+#include <sys/syscall.h>
+#endif
#include "qwen36_tier.h"
#include "backend_cuda.h"
#include "tier.h"
@@ -82,8 +86,56 @@ static void stage(uint8_t *dw, float *dsc,
memcpy(dsc+2*G.sc_gu, ds, G.sc_d *sizeof(float));
}
+/* Thread affinity around the tier's own threads (Linux).
+ *
+ * With OMP_PROC_BIND set, libgomp binds the initial thread to place 0 before
+ * main() runs, and a pthread inherits the CPU mask of the thread that creates
+ * it. The uploader thread and the CUDA runtime's own threads were therefore
+ * jailed on the OpenMP master's core: every staging copy and every driver
+ * call competed with the master thread's share of each expert matmul, and
+ * the whole team waited for it. Measured on Qwen3.8 (12 threads, one card):
+ * the CPU time per remaining expert rose 64 % while the tier was on, eating
+ * the whole gain of computing 45-59 % of the experts on the GPU. So the tier
+ * widens the calling thread's mask to every online CPU while it creates its
+ * thread and initializes CUDA, and restores the caller's mask afterwards.
+ * Raw syscalls, no _GNU_SOURCE: this file is also #included by tests after
+ * the engine's own headers. */
+#ifdef __linux__
+#define QT_AFF_WORDS 64 /* 4096 CPUs */
+typedef struct { unsigned long w[QT_AFF_WORDS]; int len; } qt_affmask;
+static int qt_aff_get(qt_affmask *m){
+ long r=syscall(SYS_sched_getaffinity,0,sizeof m->w,m->w);
+ if(r<=0) return 0;
+ m->len=(int)r; return 1;
+}
+static void qt_aff_widen(const qt_affmask *saved){
+ if(!saved->len) return;
+ long n=sysconf(_SC_NPROCESSORS_ONLN);
+ if(n<=1) return;
+ qt_affmask all; memset(&all,0,sizeof all);
+ for(long i=0;i<n && i<(long)(8*sizeof all.w);i++) all.w[i/(8*sizeof(unsigned long))] |= 1ul<<(i%(8*sizeof(unsigned long)));
+ syscall(SYS_sched_setaffinity,0,(size_t)saved->len,all.w);
+}
+static void qt_aff_restore(const qt_affmask *saved){
+ if(saved->len) syscall(SYS_sched_setaffinity,0,(size_t)saved->len,saved->w);
+}
+static int qt_aff_count_self(void){
+ qt_affmask m; if(!qt_aff_get(&m)) return 0;
+ int c=0; for(int i=0;i<m.len/(int)sizeof(unsigned long);i++) c+=__builtin_popcountl(m.w[i]);
+ return c;
+}
+#else
+typedef struct { int len; } qt_affmask;
+static int qt_aff_get(qt_affmask *m){ m->len=0; return 0; }
+static void qt_aff_widen(const qt_affmask *m){ (void)m; }
+static void qt_aff_restore(const qt_affmask *m){ (void)m; }
+static int qt_aff_count_self(void){ return 0; }
+#endif
+static int G_uploader_cpus; /* CPUs the uploader thread may run on (0 = unknown) */
+
static void *uploader(void *arg){
(void)arg;
+ G_uploader_cpus=qt_aff_count_self();
for(;;){
pthread_mutex_lock(&G.mx);
while(G.qn==0 && !G.th_stop) pthread_cond_wait(&G.cv,&G.mx);
@@ -459,7 +511,10 @@ int qt_init(int nl, int ne, int D, int Ih, int cap, int topk, int expert_gs,
}
}
if(G.ndev<1){ fprintf(stderr,"[qtier] no visible CUDA devices -> CPU path\n"); return 0; }
- if(!coli_cuda_init(G.dev,G.ndev)){ fprintf(stderr,"[qtier] coli_cuda_init failed -> CPU path\n"); return 0; }
+ qt_affmask aff; qt_aff_get(&aff); qt_aff_widen(&aff); /* CUDA's threads are born here */
+ int cuda_ok_=coli_cuda_init(G.dev,G.ndev);
+ qt_aff_restore(&aff);
+ if(!cuda_ok_){ fprintf(stderr,"[qtier] coli_cuda_init failed -> CPU path\n"); return 0; }
int have=coli_cuda_device_count();
if(have<G.ndev){ G.ndev=have; }
if(G.ndev<1){ fprintf(stderr,"[qtier] no CUDA devices -> CPU path\n"); return 0; }
@@ -623,7 +678,10 @@ int qt_init(int nl, int ne, int D, int Ih, int cap, int topk, int expert_gs,
G.is_x=malloc(G.is_x_floats*sizeof(float));
if(!G.is_x) return 0;
pthread_mutex_init(&G.mx,NULL); pthread_cond_init(&G.cv,NULL); pthread_cond_init(&G.cv_take,NULL);
- if(pthread_create(&G.th,NULL,uploader,NULL)!=0) return 0;
+ qt_aff_get(&aff); qt_aff_widen(&aff); /* the uploader inherits this mask */
+ int th_ok=pthread_create(&G.th,NULL,uploader,NULL)==0;
+ qt_aff_restore(&aff);
+ if(!th_ok) return 0;
G.on=1;
fprintf(stderr,"[qtier] CUDA VRAM expert tier active: %d device(s), %.2f MB/expert\n",
G.ndev, G.exp_bytes/1048576.0);
diff --git a/c/tests/test_qwen36_tier_fp8.c b/c/tests/test_qwen36_tier_fp8.c
index 5abc8723..6b7689cd 100644
--- a/c/tests/test_qwen36_tier_fp8.c
+++ b/c/tests/test_qwen36_tier_fp8.c
@@ -65,6 +65,11 @@ int main(void) {
/* ---- 1. the mode starts with cap < n_experts, publishes the LUT, sizes fmt 8 ---- */
printf(" 1. init\n");
+#ifdef __linux__
+ /* what libgomp does with OMP_PROC_BIND: the initial thread sits on one CPU */
+ unsigned long one[QT_AFF_WORDS]; memset(one, 0, sizeof one); one[0] = 1ul;
+ int bound = syscall(SYS_sched_setaffinity, 0, sizeof one, one) == 0;
+#endif
/* allowance for exactly 3 experts on the one device */
size_t exp_bytes = 3 * dev_alloc_footprint((size_t)D * IH) + 3 * dev_alloc_footprint(NSC * sizeof(float));
char gb[64]; snprintf(gb, sizeof gb, "%.15f", (double)(3 * exp_bytes + exp_bytes / 2) / 1073741824.0);
@@ -75,6 +80,13 @@ int main(void) {
check(fake_lut_published, "e4m3 LUT published to the backend before any upload");
check(G.exp_bytes == exp_bytes, "exp_bytes = three fmt-8 matrices + three scale tables at cudaMalloc granularity");
check(G.sc_gu == NSC && G.sc_d == NSC, "one scale per 128x128 block per matrix");
+#ifdef __linux__
+ if (bound && sysconf(_SC_NPROCESSORS_ONLN) > 1) {
+ for (int i = 0; i < 200 && !G_uploader_cpus; i++) { struct timespec ts = {0, 1000000}; nanosleep(&ts, NULL); }
+ check(G_uploader_cpus > 1, "uploader thread is not jailed on the caller's CPU");
+ check(qt_aff_count_self() == 1, "the caller's own mask is restored after init");
+ }
+#endif
/* ---- 2. a note uploads fmt 8, bytes intact, scales in order, pointers dropped ---- */
printf(" 2. note -> upload\n");