Skip to content

Run frontier MoE models on hardware you already own — pure C, zero deps, experts streamed from disk. Tiny engine, immense model. 🐦

C

fix(cuda): propagate Qwen expert collection failures

ZacharyZcRe9c45f65e412
diff --git a/c/Makefile b/c/Makefile
index 341ad971..e36d1cc1 100644
--- a/c/Makefile
+++ b/c/Makefile
@@ -2180,3 +2180,7 @@ tests/test_kv_prefix$(EXE): tests/test_kv_prefix.c kv_prefix.h
tests/test_kimi_request_state$(EXE): tests/test_kimi_request_state.c kimi_k3.c kv_prefix.h serve_codec.h st.h json.h tok.h tok_unicode.h tok_unicode_o200k.h compat.h quant.h idot.h omp_tune.h route_trace.h
$(CC) $(NOCUDA_CFLAGS) tests/test_kimi_request_state.c $(VK_OBJ) -o tests/test_kimi_request_state$(EXE) $(NOCUDA_LDFLAGS)
+
+# Async collection errors must drain all devices and reject partial output.
+tests/test_qwen36_tier_take_error$(EXE): tests/test_qwen36_tier_take_error.c tests/qwen36_fake_cuda.h qwen36_tier.c qwen36_tier.h backend_cuda.h compat.h
+ $(CC) $(CFLAGS) -DCOLI_CUDA $< -o $@ $(LDFLAGS)
diff --git a/c/qwen36.c b/c/qwen36.c
index 8fa7c56b..6d1a7e72 100644
--- a/c/qwen36.c
+++ b/c/qwen36.c
@@ -2383,7 +2383,10 @@ static void moe(Model *m, Layer *l, int layer, float *x, int S, float *out) {
tm_add(S, 3, tm_now()-_ts2);
}
double _q2 = tm_now();
- qt_take(qmask, val, K, out + (int64_t)s*D);
+ if(!qt_take(qmask, val, K, out + (int64_t)s*D)){
+ fprintf(stderr,"qwen36: CUDA expert collection failed at layer %d; stopping inference\n",layer);
+ exit(1);
+ }
if (tm_on() && S==1) {
extern double g_qt_iss, g_qt_cpu, g_qt_tak;
g_qt_iss += _q1-_q0; g_qt_cpu += _q2-_q1; g_qt_tak += tm_now()-_q2;
diff --git a/c/qwen36_tier.c b/c/qwen36_tier.c
index 2b2e24e5..a878076f 100644
--- a/c/qwen36_tier.c
+++ b/c/qwen36_tier.c
@@ -1109,17 +1109,25 @@ uint32_t qt_issue(int layer,const int *eids,int K,const float *x){
return mask;
}
-void qt_take(uint32_t mask,const float *val,int K,float *out){
+int qt_take(uint32_t mask,const float *val,int K,float *out){
(void)K;
- if(!G.on) return;
+ if(!G.on) return mask==0;
+ const float *result[QT_MAX_DEV]={0};
+ int ok=1;
+ /* Drain every device before deciding whether this layer is usable. */
if(mask) for(int di=0;di<G.ndev;di++){
- int c=G.is_cnt[di];
- if(!c) continue;
- const float *y=coli_cuda_expert_group_take(G.dev[di]);
- if(!y) continue;
- for(int j=0;j<c;j++){
+ if(!G.is_cnt[di]) continue;
+ result[di]=coli_cuda_expert_group_take(G.dev[di]);
+ if(!result[di]){
+ fprintf(stderr,"[qtier] dev %d: expert group result unavailable\n",G.dev[di]);
+ ok=0;
+ }
+ }
+ /* Do not publish a partial contribution when any device failed. */
+ for(int di=0;di<G.ndev;di++){
+ if(ok && result[di]) for(int j=0;j<G.is_cnt[di];j++){
float w=val[G.is_k[di][j]];
- const float *row=y+(size_t)j*G.D;
+ const float *row=result[di]+(size_t)j*G.D;
for(int d=0;d<G.D;d++) out[d]+=w*row[d];
}
G.is_cnt[di]=0;
@@ -1128,6 +1136,7 @@ void qt_take(uint32_t mask,const float *val,int K,float *out){
G.issue_open=0;
pthread_cond_broadcast(&G.cv_take);
pthread_mutex_unlock(&G.mx);
+ return ok;
}
void qt_stats(void){
diff --git a/c/qwen36_tier.h b/c/qwen36_tier.h
index 228288d6..62bc9650 100644
--- a/c/qwen36_tier.h
+++ b/c/qwen36_tier.h
@@ -114,8 +114,10 @@ void qt_note(int layer, int eid,
* the GPU. Compute the misses on the CPU, then call qt_take(). */
uint32_t qt_issue(int layer, const int *eids, int K, const float *x);
-/* Collect the GPU results and accumulate val[k]*y_k into out[hidden]. */
-void qt_take(uint32_t mask, const float *val, int K, float *out);
+/* Collect all GPU results and accumulate val[k]*y_k into out[hidden].
+ * Returns 0 on collection failure, leaving out unchanged. The caller must
+ * stop inference: experts selected by qt_issue were not computed on CPU. */
+int qt_take(uint32_t mask, const float *val, int K, float *out);
/* Warmstart: plan the full fill set (heat order, budget reserved), then any
* number of loader threads may call qt_note_planned per planned expert. */
@@ -153,7 +155,7 @@ static inline int qt_is_resident(int a,int b){(void)a;(void)b;return 0;}
static inline void qt_shutdown(void){}
static inline void qt_note(int a,int b,const uint8_t*c,const uint8_t*d,const uint8_t*e,const float*f,const float*g,const float*h){(void)a;(void)b;(void)c;(void)d;(void)e;(void)f;(void)g;(void)h;}
static inline uint32_t qt_issue(int a,const int*b,int c,const float*d){(void)a;(void)b;(void)c;(void)d;return 0;}
-static inline void qt_take(uint32_t a,const float*b,int c,float*d){(void)a;(void)b;(void)c;(void)d;}
+static inline int qt_take(uint32_t a,const float*b,int c,float*d){(void)b;(void)c;(void)d;return a==0;}
static inline int qt_plan_fill(int*a,int*b,int c){(void)a;(void)b;(void)c;return 0;}
static inline void qt_note_planned(int a,int b,const uint8_t*c,const uint8_t*d,const uint8_t*e,const float*f,const float*g,const float*h){(void)a;(void)b;(void)c;(void)d;(void)e;(void)f;(void)g;(void)h;}
static inline int qt_fill_next(int*a,int*b){(void)a;(void)b;return 0;}
diff --git a/c/qwen38_core.h b/c/qwen38_core.h
index c1620d11..e197522c 100644
--- a/c/qwen38_core.h
+++ b/c/qwen38_core.h
@@ -1992,7 +1992,10 @@ static void q38_moe_decode(Model *m,Layer *l,int layer,const float *x,int S,floa
}
/* GPU experts land after the CPU ones: same values, one more group in
* the float sum (that is the only ordering difference to a CPU run). */
- qt_take(qmask,route_gates,K,ys);
+ if(!qt_take(qmask,route_gates,K,ys)){
+ fprintf(stderr,"qwen38: CUDA expert collection failed at layer %d; stopping inference\n",layer);
+ exit(1);
+ }
for(int d=0;d<H;d++)ys[d]+=gate*shared[d];
}
rt_trace_end();
diff --git a/c/tests/test_qwen36_tier_take_error.c b/c/tests/test_qwen36_tier_take_error.c
new file mode 100644
index 00000000..e5602504
--- /dev/null
+++ b/c/tests/test_qwen36_tier_take_error.c
@@ -0,0 +1,34 @@
+/* A failed device must not silently drop an expert or publish a partial sum. */
+#include <stdio.h>
+#define coli_cuda_expert_group_take fixture_take
+#include "qwen36_fake_cuda.h"
+#undef coli_cuda_expert_group_take
+static int failed=-1, calls[2];
+static float rows[2][2]={{2,4},{8,12}};
+const float *coli_cuda_expert_group_take(int device) {
+ calls[device]++;
+ return device==failed?NULL:rows[device];
+}
+#include "../qwen36_tier.c"
+int main(void) {
+ G.on=1;G.ndev=2;G.D=2;G.dev[0]=0;G.dev[1]=1;
+ pthread_mutex_init(&G.mx,NULL);pthread_cond_init(&G.cv_take,NULL);
+ for(failed=-1;failed<2;failed++){
+ G.issue_open=1;G.is_cnt[0]=G.is_cnt[1]=1;
+ G.is_k[0][0]=0;G.is_k[1][0]=1;calls[0]=calls[1]=0;
+ float val[2]={0.5f,0.25f},out[2]={10,20};
+ int ok=qt_take(3,val,2,out);
+ float want0=failed<0?13:10,want1=failed<0?25:20;
+ if(ok!=(failed<0)||out[0]!=want0||out[1]!=want1||
+ calls[0]!=1||calls[1]!=1||G.is_cnt[0]||G.is_cnt[1]||G.issue_open){
+ fprintf(stderr,"take failed case %d: status=%d output=%g,%g calls=%d,%d\n",
+ failed,ok,out[0],out[1],calls[0],calls[1]);return 1;
+ }
+ }
+ G.issue_open=1;
+ if(!qt_take(0,NULL,0,NULL)||G.issue_open) return 1;
+ G.on=0;
+ if(!qt_take(0,NULL,0,NULL)||qt_take(1,NULL,0,NULL)) return 1;
+ pthread_cond_destroy(&G.cv_take);pthread_mutex_destroy(&G.mx);
+ puts("tier take error: ok");return 0;
+}