Run frontier MoE models on hardware you already own — pure C, zero deps, experts streamed from disk. Tiny engine, immense model. 🐦
C
fix(cuda): propagate Qwen expert collection failures
ZacharyZcRe9c45f65e412
diff --git a/c/Makefile b/c/Makefileindex 341ad971..e36d1cc1 100644--- a/c/Makefile+++ b/c/Makefile@@ -2180,3 +2180,7 @@ tests/test_kv_prefix$(EXE): tests/test_kv_prefix.c kv_prefix.htests/test_kimi_request_state$(EXE): tests/test_kimi_request_state.c kimi_k3.c kv_prefix.h serve_codec.h st.h json.h tok.h tok_unicode.h tok_unicode_o200k.h compat.h quant.h idot.h omp_tune.h route_trace.h$(CC) $(NOCUDA_CFLAGS) tests/test_kimi_request_state.c $(VK_OBJ) -o tests/test_kimi_request_state$(EXE) $(NOCUDA_LDFLAGS)++# Async collection errors must drain all devices and reject partial output.+tests/test_qwen36_tier_take_error$(EXE): tests/test_qwen36_tier_take_error.c tests/qwen36_fake_cuda.h qwen36_tier.c qwen36_tier.h backend_cuda.h compat.h+ $(CC) $(CFLAGS) -DCOLI_CUDA $< -o $@ $(LDFLAGS)diff --git a/c/qwen36.c b/c/qwen36.cindex 8fa7c56b..6d1a7e72 100644--- a/c/qwen36.c+++ b/c/qwen36.c@@ -2383,7 +2383,10 @@ static void moe(Model *m, Layer *l, int layer, float *x, int S, float *out) {tm_add(S, 3, tm_now()-_ts2);}double _q2 = tm_now();- qt_take(qmask, val, K, out + (int64_t)s*D);+ if(!qt_take(qmask, val, K, out + (int64_t)s*D)){+ fprintf(stderr,"qwen36: CUDA expert collection failed at layer %d; stopping inference\n",layer);+ exit(1);+ }if (tm_on() && S==1) {extern double g_qt_iss, g_qt_cpu, g_qt_tak;g_qt_iss += _q1-_q0; g_qt_cpu += _q2-_q1; g_qt_tak += tm_now()-_q2;diff --git a/c/qwen36_tier.c b/c/qwen36_tier.cindex 2b2e24e5..a878076f 100644--- a/c/qwen36_tier.c+++ b/c/qwen36_tier.c@@ -1109,17 +1109,25 @@ uint32_t qt_issue(int layer,const int *eids,int K,const float *x){return mask;}-void qt_take(uint32_t mask,const float *val,int K,float *out){+int qt_take(uint32_t mask,const float *val,int K,float *out){(void)K;- if(!G.on) return;+ if(!G.on) return mask==0;+ const float *result[QT_MAX_DEV]={0};+ int ok=1;+ /* Drain every device before deciding whether this layer is usable. */if(mask) for(int di=0;di<G.ndev;di++){- int c=G.is_cnt[di];- if(!c) continue;- const float *y=coli_cuda_expert_group_take(G.dev[di]);- if(!y) continue;- for(int j=0;j<c;j++){+ if(!G.is_cnt[di]) continue;+ result[di]=coli_cuda_expert_group_take(G.dev[di]);+ if(!result[di]){+ fprintf(stderr,"[qtier] dev %d: expert group result unavailable\n",G.dev[di]);+ ok=0;+ }+ }+ /* Do not publish a partial contribution when any device failed. */+ for(int di=0;di<G.ndev;di++){+ if(ok && result[di]) for(int j=0;j<G.is_cnt[di];j++){float w=val[G.is_k[di][j]];- const float *row=y+(size_t)j*G.D;+ const float *row=result[di]+(size_t)j*G.D;for(int d=0;d<G.D;d++) out[d]+=w*row[d];}G.is_cnt[di]=0;@@ -1128,6 +1136,7 @@ void qt_take(uint32_t mask,const float *val,int K,float *out){G.issue_open=0;pthread_cond_broadcast(&G.cv_take);pthread_mutex_unlock(&G.mx);+ return ok;}void qt_stats(void){diff --git a/c/qwen36_tier.h b/c/qwen36_tier.hindex 228288d6..62bc9650 100644--- a/c/qwen36_tier.h+++ b/c/qwen36_tier.h@@ -114,8 +114,10 @@ void qt_note(int layer, int eid,* the GPU. Compute the misses on the CPU, then call qt_take(). */uint32_t qt_issue(int layer, const int *eids, int K, const float *x);-/* Collect the GPU results and accumulate val[k]*y_k into out[hidden]. */-void qt_take(uint32_t mask, const float *val, int K, float *out);+/* Collect all GPU results and accumulate val[k]*y_k into out[hidden].+ * Returns 0 on collection failure, leaving out unchanged. The caller must+ * stop inference: experts selected by qt_issue were not computed on CPU. */+int qt_take(uint32_t mask, const float *val, int K, float *out);/* Warmstart: plan the full fill set (heat order, budget reserved), then any* number of loader threads may call qt_note_planned per planned expert. */@@ -153,7 +155,7 @@ static inline int qt_is_resident(int a,int b){(void)a;(void)b;return 0;}static inline void qt_shutdown(void){}static inline void qt_note(int a,int b,const uint8_t*c,const uint8_t*d,const uint8_t*e,const float*f,const float*g,const float*h){(void)a;(void)b;(void)c;(void)d;(void)e;(void)f;(void)g;(void)h;}static inline uint32_t qt_issue(int a,const int*b,int c,const float*d){(void)a;(void)b;(void)c;(void)d;return 0;}-static inline void qt_take(uint32_t a,const float*b,int c,float*d){(void)a;(void)b;(void)c;(void)d;}+static inline int qt_take(uint32_t a,const float*b,int c,float*d){(void)b;(void)c;(void)d;return a==0;}static inline int qt_plan_fill(int*a,int*b,int c){(void)a;(void)b;(void)c;return 0;}static inline void qt_note_planned(int a,int b,const uint8_t*c,const uint8_t*d,const uint8_t*e,const float*f,const float*g,const float*h){(void)a;(void)b;(void)c;(void)d;(void)e;(void)f;(void)g;(void)h;}static inline int qt_fill_next(int*a,int*b){(void)a;(void)b;return 0;}diff --git a/c/qwen38_core.h b/c/qwen38_core.hindex c1620d11..e197522c 100644--- a/c/qwen38_core.h+++ b/c/qwen38_core.h@@ -1992,7 +1992,10 @@ static void q38_moe_decode(Model *m,Layer *l,int layer,const float *x,int S,floa}/* GPU experts land after the CPU ones: same values, one more group in* the float sum (that is the only ordering difference to a CPU run). */- qt_take(qmask,route_gates,K,ys);+ if(!qt_take(qmask,route_gates,K,ys)){+ fprintf(stderr,"qwen38: CUDA expert collection failed at layer %d; stopping inference\n",layer);+ exit(1);+ }for(int d=0;d<H;d++)ys[d]+=gate*shared[d];}rt_trace_end();diff --git a/c/tests/test_qwen36_tier_take_error.c b/c/tests/test_qwen36_tier_take_error.cnew file mode 100644index 00000000..e5602504--- /dev/null+++ b/c/tests/test_qwen36_tier_take_error.c@@ -0,0 +1,34 @@+/* A failed device must not silently drop an expert or publish a partial sum. */+#include <stdio.h>+#define coli_cuda_expert_group_take fixture_take+#include "qwen36_fake_cuda.h"+#undef coli_cuda_expert_group_take+static int failed=-1, calls[2];+static float rows[2][2]={{2,4},{8,12}};+const float *coli_cuda_expert_group_take(int device) {+ calls[device]++;+ return device==failed?NULL:rows[device];+}+#include "../qwen36_tier.c"+int main(void) {+ G.on=1;G.ndev=2;G.D=2;G.dev[0]=0;G.dev[1]=1;+ pthread_mutex_init(&G.mx,NULL);pthread_cond_init(&G.cv_take,NULL);+ for(failed=-1;failed<2;failed++){+ G.issue_open=1;G.is_cnt[0]=G.is_cnt[1]=1;+ G.is_k[0][0]=0;G.is_k[1][0]=1;calls[0]=calls[1]=0;+ float val[2]={0.5f,0.25f},out[2]={10,20};+ int ok=qt_take(3,val,2,out);+ float want0=failed<0?13:10,want1=failed<0?25:20;+ if(ok!=(failed<0)||out[0]!=want0||out[1]!=want1||+ calls[0]!=1||calls[1]!=1||G.is_cnt[0]||G.is_cnt[1]||G.issue_open){+ fprintf(stderr,"take failed case %d: status=%d output=%g,%g calls=%d,%d\n",+ failed,ok,out[0],out[1],calls[0],calls[1]);return 1;+ }+ }+ G.issue_open=1;+ if(!qt_take(0,NULL,0,NULL)||G.issue_open) return 1;+ G.on=0;+ if(!qt_take(0,NULL,0,NULL)||qt_take(1,NULL,0,NULL)) return 1;+ pthread_cond_destroy(&G.cv_take);pthread_mutex_destroy(&G.mx);+ puts("tier take error: ok");return 0;+}