Skip to content

Run frontier MoE models on hardware you already own — pure C, zero deps, experts streamed from disk. Tiny engine, immense model. 🐦

C

router: guard best==-1 in top-K selection (NaN logits caused OOB read/writes)

刘宣岐6dc1ee048cab
diff --git a/c/Makefile b/c/Makefile
index c7175681..c61d6485 100644
--- a/c/Makefile
+++ b/c/Makefile
@@ -199,7 +199,7 @@ else
PYTHON ?= python3
endif
CUDA_OBJ =
-TEST_BINS = tests/test_json$(EXE) tests/test_st$(EXE) tests/test_st_mirror$(EXE) tests/test_st_pread$(EXE) tests/test_tier$(EXE) tests/test_grammar$(EXE) tests/test_schema_gbnf$(EXE) tests/test_decode_batch$(EXE) tests/test_idot$(EXE) tests/test_i4_grouped$(EXE) tests/test_stops$(EXE) tests/test_topp$(EXE) tests/test_sample_nan$(EXE) tests/test_temp_env$(EXE) tests/test_tok_o200k$(EXE) tests/test_kv_alloc$(EXE) tests/test_int3$(EXE) tests/test_int3_load$(EXE) tests/test_i4_acc512$(EXE) tests/test_compat_direct$(EXE) tests/test_dsa_select$(EXE) tests/test_logit_nan$(EXE) tests/test_pipe_block$(EXE) tests/test_e8_kernel$(EXE)
+TEST_BINS = tests/test_json$(EXE) tests/test_st$(EXE) tests/test_st_mirror$(EXE) tests/test_st_pread$(EXE) tests/test_tier$(EXE) tests/test_grammar$(EXE) tests/test_schema_gbnf$(EXE) tests/test_decode_batch$(EXE) tests/test_idot$(EXE) tests/test_i4_grouped$(EXE) tests/test_stops$(EXE) tests/test_topp$(EXE) tests/test_sample_nan$(EXE) tests/test_temp_env$(EXE) tests/test_tok_o200k$(EXE) tests/test_kv_alloc$(EXE) tests/test_int3$(EXE) tests/test_int3_load$(EXE) tests/test_i4_acc512$(EXE) tests/test_compat_direct$(EXE) tests/test_dsa_select$(EXE) tests/test_logit_nan$(EXE) tests/test_router_nan$(EXE) tests/test_pipe_block$(EXE) tests/test_e8_kernel$(EXE)
ifneq (,$(LINUX))
TEST_BINS += tests/test_uring$(EXE)
endif
@@ -435,6 +435,9 @@ tests/test_int3_load$(EXE): tests/test_int3_load.c colibri.c st.h uring.h json.h
tests/test_logit_nan$(EXE): tests/test_logit_nan.c colibri.c st.h uring.h json.h tok.h tok_unicode.h compat.h grammar.h tier.h
$(CC) $(CFLAGS) $< -o $@ $(LDFLAGS)
+tests/test_router_nan$(EXE): tests/test_router_nan.c colibri.c st.h uring.h json.h tok.h tok_unicode.h compat.h grammar.h tier.h
+ $(CC) $(CFLAGS) $< -o $@ $(LDFLAGS)
+
tests/test_i4_acc512$(EXE): tests/test_i4_acc512.c
$(CC) $(CFLAGS) $< -o $@ $(LDFLAGS)
diff --git a/c/colibri.c b/c/colibri.c
index bb24c831..dfc05932 100644
--- a/c/colibri.c
+++ b/c/colibri.c
@@ -2715,6 +2715,22 @@ static int expert_is_resident(Model *m, int layer, int eid){
return 0;
}
+/* I loop di selezione top-K partono da best=-1 e lo usano come indice appena il
+ * giro finisce. Se OGNI punteggio candidato e' non finito nessun confronto riesce
+ * (NaN>bv e' falso) e best resta -1: logit[-1] e' una lettura fuori range, e
+ * idx[]=-1 si propaga a eusage/eheat/elast (tre scritture heap a indice -1) e al
+ * VLA seen[] (underflow di stack). Non e' teorico: c/tests/test_logit_nan.c
+ * documenta NaN/Inf nei logit da un tile expert corrotto o da un overflow fp al
+ * confine di eviction, e ha indurito solo il lato sampling — il router e' a monte.
+ * Qui degradiamo a una scelta deterministica e avvisiamo una volta sola. */
+static int router_best_or_fallback(int best, int kk, int E, int layer){
+ if(best>=0) return best;
+ static int warned;
+ if(!warned){ warned=1;
+ fprintf(stderr,"[router] logits non finiti al layer %d: selezione degradata\n",layer); }
+ return kk<E ? kk : 0;
+}
+
static void moe(Model *m, Layer *l, int layer, float *x, int S, float *out, int with_shared){
if(g_pilot_real){ /* barriera cross-layer: prendi possesso di QUESTO layer e aspetta
* l'eventuale load-pilota in volo sullo stesso layer (dopodiche' il
@@ -2761,6 +2777,23 @@ static void moe(Model *m, Layer *l, int layer, float *x, int S, float *out, int
memcpy(idxs,g_pre_idx,(size_t)S*K*sizeof(int));
memcpy(ws,g_pre_w,(size_t)S*K*sizeof(float));
memcpy(keff,g_pre_keff,(size_t)S*sizeof(int));
+ /* Stessa classe di difetto dal lato GPU: un risultato di routing malformato
+ * (keff oltre K, id expert fuori range) indicizzerebbe gli stessi array
+ * eusage/eheat/elast e il VLA seen[] di FASE B. Meglio sanificare qui una
+ * volta che fidarsi del device. */
+ for(int s=0;s<S;s++){
+ if(keff[s]<0) keff[s]=0;
+ if(keff[s]>K) keff[s]=K;
+ for(int kk=0;kk<keff[s];kk++){
+ int e=idxs[(int64_t)s*K+kk];
+ if(e<0||e>=E){
+ static int warned_pre;
+ if(!warned_pre){ warned_pre=1;
+ fprintf(stderr,"[router] indice expert %d fuori range al layer %d: azzerato\n",e,layer); }
+ idxs[(int64_t)s*K+kk]=0;
+ }
+ }
+ }
for(int s=0;s<S;s++){
m->ereq+=keff[s];
for(int kk=0;kk<keff[s];kk++){
@@ -2802,6 +2835,7 @@ static void moe(Model *m, Layer *l, int layer, float *x, int S, float *out, int
for(int kk=0;kk<Mmax;kk++){ int best=-1; float bv=-1e30f;
for(int e=0;e<E;e++){ int tk=0; for(int j=0;j<kk;j++) if(rank_buf[j]==e){tk=1;break;}
if(!tk && choice[e]>bv){bv=choice[e];best=e;} }
+ best=router_best_or_fallback(best,kk,E,layer);
rank_buf[kk]=best; rank_w[kk]=logit[best];
}
float tot=1e-20f; for(int kk=0;kk<Mmax;kk++) tot+=rank_w[kk]>0?rank_w[kk]:0;
@@ -2813,6 +2847,7 @@ static void moe(Model *m, Layer *l, int layer, float *x, int S, float *out, int
for(int kk=0;kk<Mwin;kk++){ int best=-1; float bv=-1e30f;
for(int e=0;e<E;e++){ int tk=0; for(int j=0;j<kk;j++) if(rank_buf[j]==e){tk=1;break;}
if(!tk && choice[e]>bv){bv=choice[e];best=e;} }
+ best=router_best_or_fallback(best,kk,E,layer);
rank_buf[kk]=best; rank_w[kk]=logit[best];
}
}
@@ -2881,6 +2916,7 @@ static void moe(Model *m, Layer *l, int layer, float *x, int S, float *out, int
for(int kk=0;kk<Ksel;kk++){ int best=-1; float bv=-1e30f;
for(int e=0;e<E;e++){ int tk=0; for(int j=0;j<kk;j++) if(idx[j]==e){tk=1;break;}
if(!tk && choice[e]>bv){bv=choice[e];best=e;} }
+ best=router_best_or_fallback(best,kk,E,layer);
idx[kk]=best; w[kk]=logit[best];
}
if(g_route_agree){
diff --git a/c/tests/test_router_nan.c b/c/tests/test_router_nan.c
new file mode 100644
index 00000000..0f439e78
--- /dev/null
+++ b/c/tests/test_router_nan.c
@@ -0,0 +1,79 @@
+/* Regression for non-finite-logit poisoning of the ROUTER (upstream of sampling).
+ *
+ * test_logit_nan.c hardened the sampling side. The routing side was not, and it
+ * runs first: the three top-K selection loops in moe() start from best=-1 and use
+ * it as an index as soon as the scan ends. choice[e] = sigmoidf(logit[e]) + bias,
+ * and sigmoidf(NaN) is NaN, so `choice[e] > bv` is false for every e — best stays
+ * -1. The old code then did idx[kk]=best; w[kk]=logit[best], which is:
+ *
+ * - a heap OOB read of logits_all[s*E - 1] at s==0;
+ * - three OOB writes at index -1 (eusage / eheat / elast);
+ * - a stack buffer underflow on the VLA `unsigned char seen[E]` in FASE B;
+ * - uniq[nu++] = -1 handed to expert_load().
+ *
+ * Fix under test: router_best_or_fallback() maps a failed scan to a deterministic
+ * in-range expert and warns once. Degrade + diagnose, never index out of range.
+ *
+ * No model file needed: exercises the guard and the exact scan that trips it. */
+#include <assert.h>
+#include <math.h>
+#define main coli_glm_main_unused
+#include "../colibri.c"
+#undef main
+
+/* the selection scan as written in moe(), verbatim in shape */
+static int scan_best(const float *choice, int E, const int *taken, int ntaken){
+ int best=-1; float bv=-1e30f;
+ for(int e=0;e<E;e++){
+ int tk=0; for(int j=0;j<ntaken;j++) if(taken[j]==e){tk=1;break;}
+ if(!tk && choice[e]>bv){bv=choice[e];best=e;}
+ }
+ return best;
+}
+
+int main(void){
+ const int E=8;
+
+ /* --- the trigger is real: an all-NaN choice[] leaves best==-1 --- */
+ { float choice[8]; for(int e=0;e<E;e++) choice[e]=sigmoidf(NAN)+0.0f;
+ for(int e=0;e<E;e++) assert(choice[e]!=choice[e] && "sigmoidf(NaN) is NaN");
+ assert(scan_best(choice,E,NULL,0)==-1 && "all-NaN scan selects nothing"); }
+
+ /* one NaN logit is enough to poison a slot once the finite ones are taken */
+ { float choice[8]={NAN,NAN,NAN,NAN,NAN,NAN,NAN,2.f};
+ int taken[1]={7};
+ assert(scan_best(choice,E,taken,1)==-1 && "remaining candidates all NaN"); }
+
+ /* --- the guard: passthrough when the scan succeeded --- */
+ assert(router_best_or_fallback(5,0,E,0)==5);
+ assert(router_best_or_fallback(0,3,E,0)==0 && "expert 0 is a valid pick, not a failure");
+
+ /* --- the guard: a failed scan degrades to a deterministic in-range expert --- */
+ for(int kk=0;kk<E;kk++){
+ int b=router_best_or_fallback(-1,kk,E,0);
+ assert(b>=0 && b<E && "fallback must be a valid expert index");
+ assert(b==kk && "distinct kk -> distinct expert, so a slot is never duplicated");
+ }
+ /* Ksel may exceed E on a malformed config: still in range, never -1 */
+ for(int kk=E;kk<E+4;kk++){
+ int b=router_best_or_fallback(-1,kk,E,0);
+ assert(b>=0 && b<E && "fallback stays in range past E");
+ }
+
+ /* --- end to end: the accounting arrays are indexed safely after the guard --- */
+ { float choice[8]; for(int e=0;e<E;e++) choice[e]=NAN;
+ unsigned char seen[8]; memset(seen,0,sizeof seen);
+ int idx[4]; int Ksel=4;
+ for(int kk=0;kk<Ksel;kk++){
+ int best=scan_best(choice,E,idx,kk);
+ best=router_best_or_fallback(best,kk,E,0);
+ idx[kk]=best;
+ }
+ for(int kk=0;kk<Ksel;kk++){
+ assert(idx[kk]>=0 && idx[kk]<E && "no OOB index reaches eusage/eheat/elast");
+ seen[idx[kk]]=1; /* would have been seen[-1] before the fix */
+ } }
+
+ printf("OK test_router_nan: top-K selection guards best==-1\n");
+ return 0;
+}