Run frontier MoE models on hardware you already own — pure C, zero deps, experts streamed from disk. Tiny engine, immense model. 🐦
C
Don't trust a converted checkpoint's config for the stop set
JustVugg6de32c55f69a
diff --git a/c/Makefile b/c/Makefileindex 7fb9b02a..97362bb3 100644--- a/c/Makefile+++ b/c/Makefile@@ -166,7 +166,7 @@ elsePYTHON ?= python3endifCUDA_OBJ =-TEST_BINS = tests/test_json$(EXE) tests/test_st$(EXE) tests/test_tier$(EXE) tests/test_grammar$(EXE) tests/test_schema_gbnf$(EXE) tests/test_decode_batch$(EXE) tests/test_idot$(EXE) tests/test_i4_grouped$(EXE) tests/test_kv_alloc$(EXE) tests/test_i4_acc512$(EXE) tests/test_compat_direct$(EXE)+TEST_BINS = tests/test_json$(EXE) tests/test_st$(EXE) tests/test_tier$(EXE) tests/test_grammar$(EXE) tests/test_schema_gbnf$(EXE) tests/test_decode_batch$(EXE) tests/test_idot$(EXE) tests/test_i4_grouped$(EXE) tests/test_stops$(EXE) tests/test_kv_alloc$(EXE) tests/test_i4_acc512$(EXE) tests/test_compat_direct$(EXE)ifneq (,$(LINUX))TEST_BINS += tests/test_uring$(EXE)endif@@ -300,6 +300,9 @@ tests/test_idot$(EXE): tests/test_idot.c glm.c st.h uring.h json.h tok.h tok_unitests/test_i4_grouped$(EXE): tests/test_i4_grouped.c glm.c st.h uring.h json.h tok.h tok_unicode.h compat.h grammar.h tier.h$(CC) $(CFLAGS) $< -o $@ $(LDFLAGS)+tests/test_stops$(EXE): tests/test_stops.c glm.c st.h uring.h json.h tok.h tok_unicode.h compat.h grammar.h tier.h+ $(CC) $(CFLAGS) $< -o $@ $(LDFLAGS)+tests/test_kv_alloc$(EXE): tests/test_kv_alloc.c glm.c st.h json.h tok.h tok_unicode.h compat.h grammar.h tier.h$(CC) $(CFLAGS) $< -o $@ $(LDFLAGS)diff --git a/c/glm.c b/c/glm.cindex 00c70768..de80aa29 100644--- a/c/glm.c+++ b/c/glm.c@@ -1277,6 +1277,35 @@ static void load_cfg(Cfg *c, const char *snap){if(eo){ if(eo->t==J_NUM) c->stop_ids[c->n_stop++]=(int)eo->num;else if(eo->t==J_ARR) for(int i=0;i<eo->len && c->n_stop<8;i++)c->stop_ids[c->n_stop++]=(int)eo->kids[i]->num; }+ /* generation_config.json e' il file AUTOREVOLE per la generazione secondo HuggingFace:+ * config.json ne porta spesso una copia legacy o parziale. Un tool di conversione che+ * rigenera un config.json ridotto lascia il motore fermo su MENO stop del dovuto, e i+ * token di controllo che restano finiscono stampati in chat come testo (woolcoxm, #298:+ * "the stop token being printed to chat", verificato sui token id). Unione dei due:+ * uno stop in piu' non fa danno, uno in meno si' -- e chi converte i pesi non siamo noi.+ * EN: generation_config.json is HF's authority for generation; config.json often carries+ * a partial legacy copy. Union both -- an extra stop is harmless, a missing one is not. */+ { char gp[2100]; snprintf(gp,sizeof(gp),"%s/generation_config.json",snap);+ FILE *gf=fopen(gp,"rb"); /* assente = nessun problema: e' opzionale */+ if(gf){+ fseek(gf,0,SEEK_END); long gn=ftell(gf); fseek(gf,0,SEEK_SET);+ if(gn>0){+ char *gb=malloc(gn+1); size_t gg=fread(gb,1,gn,gf); gb[gg]=0;+ char *ga=NULL; jval *gr=json_parse(gb,&ga);+ jval *ge=gr?json_get(gr,"eos_token_id"):NULL;+ if(ge){+ int add[8], na=0;+ if(ge->t==J_NUM) add[na++]=(int)ge->num;+ else if(ge->t==J_ARR) for(int i=0;i<ge->len && na<8;i++) add[na++]=(int)ge->kids[i]->num;+ for(int i=0;i<na && c->n_stop<8;i++){+ int dup=0; for(int j=0;j<c->n_stop;j++) if(c->stop_ids[j]==add[i]) dup=1;+ if(!dup) c->stop_ids[c->n_stop++]=add[i];+ }+ }+ free(ga); free(gb);+ }+ fclose(gf);+ } }/* DSA lightning indexer: parametri + tipo per-layer (lista esplicita o formula freq/offset) */c->index_topk=gi(r,"index_topk"); c->index_nh=gi(r,"index_n_heads"); c->index_hd=gi(r,"index_head_dim");{ jval *it=json_get(r,"indexer_types");@@ -4107,18 +4136,34 @@ static int pick_tok(const float *lo, int V, int ban){/* stop-set attivo (popolato da run_text/run_serve dal config; vuoto in validazione,* dove si genera un numero fisso di token da confrontare con l'oracolo) */-static int g_stop[9], g_nstop=0;+static int g_stop[64], g_nstop=0; /* config eos + ogni added-token "special" del tokenizer */static void repin_pass_limit(Model *m,int limit);static void repin_pass(Model *m){ repin_pass_limit(m,16); }static inline int is_stop(int t){ for(int i=0;i<g_nstop;i++) if(t==g_stop[i]) return 1; return 0; }-static void stops_arm(const Cfg *c, int tok_eos){+/* T=NULL -> solo gli stop del config (validazione/oracolo, dove il tokenizer non serve). */+static void stops_arm_tok(const Cfg *c, int tok_eos, Tok *T){g_nstop=0;- for(int i=0;i<c->n_stop;i++) g_stop[g_nstop++]=c->stop_ids[i];- if(tok_eos>=0 && !is_stop(tok_eos)) g_stop[g_nstop++]=tok_eos;+ for(int i=0;i<c->n_stop && g_nstop<64;i++) g_stop[g_nstop++]=c->stop_ids[i];+ if(tok_eos>=0 && !is_stop(tok_eos) && g_nstop<64) g_stop[g_nstop++]=tok_eos;+ int nsp=0;+ /* DIFESA IN PROFONDITA' (woolcoxm, #298): il tokenizer marca "special":true i token di+ * CONTROLLO -- <|user|>, <|assistant|>, <|observation|>, <sop>, [gMASK], i marker+ * image/video/audio. Nessuno di questi e' contenuto legittimo di una risposta: se il+ * modello ne emette uno, il turno e' finito (infatti GLM ne elenca tre fra gli eos+ * ufficiali). Senza questo, uno di quei token non elencato nel config veniva+ * DETOKENIZZATO E STAMPATO IN CHAT come testo, e la generazione proseguiva oltre la+ * fine reale -- l'"added stuff on the end" riportato su un checkpoint convertito.+ * Fidarsi del config di pesi convertiti da terzi e' precisamente cio' che non+ * possiamo controllare; il flag del tokenizer lo possiamo leggere.+ * NB: <think>/<tool_call>/<arg_key> hanno "special":false e restano contenuto vero. */+ if(T) for(int id=0; id<T->n_ids && g_nstop<64; id++)+ if(T->id_special[id] && !is_stop(id)){ g_stop[g_nstop++]=id; nsp++; }fprintf(stderr,"[stop] %d stop tokens:",g_nstop);for(int i=0;i<g_nstop;i++) fprintf(stderr," %d",g_stop[i]);+ if(nsp) fprintf(stderr," (%d from the tokenizer's special set)",nsp);fprintf(stderr,"\n");}+static void stops_arm(const Cfg *c, int tok_eos){ stops_arm_tok(c,tok_eos,NULL); }/* decode greedy con SELF-SPECULATION n-gram: LOSSLESS (output identico al greedy puro).* Ogni forward verifica fino a g_draft token proposti dal contesto: i token accettati@@ -4399,7 +4444,7 @@ static void run_text(Model *m, const char *snap, const char *prompt, int ngen){Cfg *c=&m->c; char tkp[2048]; snprintf(tkp,sizeof(tkp),"%s/tokenizer.json",snap);Tok T; tok_load(&T,tkp);int eos=tok_id_of(&T,"<|endoftext|>");- stops_arm(&m->c, eos);+ stops_arm_tok(&m->c, eos, &T);grammar_setup(&T); /* metodo F: GRAMMAR=file.gbnf (#48) */if(g_temp<0) g_temp=0.7f; /* auto: 0.7, NON l'1.0 ufficiale — la coda della* distribuzione int4 e' rumore di quantizzazione */@@ -4869,7 +4914,7 @@ static int mux_submit(Model *m, Tok *T, ServeCtx *ctx, ServeReq *req, int nctx,static void run_serve_mux(Model *m, const char *snap){char tkp[2048]; snprintf(tkp,sizeof(tkp),"%s/tokenizer.json",snap);- Tok T; tok_load(&T,tkp); int eos=tok_id_of(&T,"<|endoftext|>"); stops_arm(&m->c,eos);+ Tok T; tok_load(&T,tkp); int eos=tok_id_of(&T,"<|endoftext|>"); stops_arm_tok(&m->c,eos,&T);g_draft=0; /* one scheduler owns every forward; MTP/speculation is not ragged-safe */int maxctx=getenv("CTX")?atoi(getenv("CTX")):4096;int nctx=getenv("KV_SLOTS")?atoi(getenv("KV_SLOTS")):1;@@ -4970,7 +5015,7 @@ static void run_serve(Model *m, const char *snap){char tkp[2048]; snprintf(tkp,sizeof(tkp),"%s/tokenizer.json",snap);Tok T; tok_load(&T,tkp);int eos=tok_id_of(&T,"<|endoftext|>");- stops_arm(&m->c, eos);+ stops_arm_tok(&m->c, eos, &T);grammar_setup(&T); /* metodo F: GRAMMAR=file.gbnf (#48) */if(g_temp<0) g_temp=0.7f; /* auto: 0.7, NON l'1.0 ufficiale — la coda della* distribuzione int4 e' rumore di quantizzazione */diff --git a/c/tests/test_stops.c b/c/tests/test_stops.cnew file mode 100644index 00000000..a24215ff--- /dev/null+++ b/c/tests/test_stops.c@@ -0,0 +1,132 @@+/* Stop-token arming: the engine must not depend on a converted checkpoint's+ * config.json being complete.+ *+ * Why this test exists (#298, #307): GLM-5.2 declares THREE eos ids+ * (<|endoftext|>, <|user|>, <|observation|>). A conversion tool that rewrites+ * config.json with a reduced eos_token_id list leaves the engine stopping on+ * fewer tokens than the model actually emits -- and the ones it misses get+ * DETOKENIZED AND PRINTED INTO THE CHAT as literal text, with generation+ * continuing past the real end of the turn. @woolcoxm hit exactly this on a+ * g64 checkpoint and confirmed it by comparing token ids.+ *+ * Two independent defenses, one test each:+ * 1. eos_token_id is unioned from generation_config.json (HuggingFace's+ * authority for generation) as well as config.json.+ * 2. every added-token the TOKENIZER marks "special":true is a stop, whatever+ * the configs say. Those are control tokens (<|user|>, <sop>, ...) and are+ * never legitimate content. <think>/<tool_call> are "special":false and+ * must NOT be swept up -- they are real output.+ *+ * Defense 2 is what makes this robust against checkpoints we don't control:+ * even with BOTH configs mutilated, a control token cannot leak into a reply. */+#define main coli_glm_main_unused+#include "../glm.c"+#undef main++static const char *TOKJSON =+"{\"model\":{\"vocab\":{\"a\":0,\"b\":1,\"c\":2},\"merges\":[[\"a\",\"b\"]]},"+" \"added_tokens\":["+" {\"id\":100,\"content\":\"<|endoftext|>\",\"special\":true},"+" {\"id\":101,\"content\":\"<|user|>\",\"special\":true},"+" {\"id\":102,\"content\":\"<|observation|>\",\"special\":true},"+" {\"id\":103,\"content\":\"<|assistant|>\",\"special\":true},"+" {\"id\":104,\"content\":\"<sop>\",\"special\":true},"+" {\"id\":110,\"content\":\"<think>\",\"special\":false},"+" {\"id\":111,\"content\":\"<tool_call>\",\"special\":false}"+"]}";++/* minimal config.json that survives load_cfg's range validation */+static void write_cfg(const char *dir, const char *fname, const char *eos_json){+ char p[512]; snprintf(p,sizeof(p),"%s/%s",dir,fname);+ FILE *f=fopen(p,"w"); if(!f){ perror(p); exit(1); }+ fprintf(f,"{\"hidden_size\":64,\"num_hidden_layers\":2,\"num_attention_heads\":4,"+ "\"n_routed_experts\":8,\"num_experts_per_tok\":2,\"moe_intermediate_size\":32,"+ "\"intermediate_size\":64,\"first_k_dense_replace\":1,\"q_lora_rank\":0,"+ "\"kv_lora_rank\":16,\"qk_nope_head_dim\":8,\"qk_rope_head_dim\":8,"+ "\"v_head_dim\":8,\"n_shared_experts\":1,\"vocab_size\":200,"+ "\"n_group\":1,\"topk_group\":1,\"rope_theta\":10000.0");+ if(eos_json) fprintf(f,",\"eos_token_id\":%s",eos_json);+ fprintf(f,"}\n"); fclose(f);+}+static void write_tok(const char *dir){+ char p[512]; snprintf(p,sizeof(p),"%s/tokenizer.json",dir);+ FILE *f=fopen(p,"w"); if(!f){ perror(p); exit(1); }+ fputs(TOKJSON,f); fclose(f);+}+static void rm_file(const char *dir, const char *fname){+ char p[512]; snprintf(p,sizeof(p),"%s/%s",dir,fname); remove(p);+}++static int expect(const char *what, int id, int want){+ int got=is_stop(id);+ if(got!=want){ fprintf(stderr," FAIL %s: token %d stop=%d, expected %d\n",what,id,got,want); return 1; }+ return 0;+}++int main(void){+ int fail=0;+ char dir[]="/tmp/coli_stops_XXXXXX";+ if(!mkdtemp(dir)){ perror("mkdtemp"); return 1; }+ write_tok(dir);+ char tkp[512]; snprintf(tkp,sizeof(tkp),"%s/tokenizer.json",dir);+ Tok T; tok_load(&T,tkp);+ printf("test_stops: stop arming vs incomplete checkpoint metadata\n");++ /* 1. the tokenizer's "special" flag must survive loading, and must NOT+ * swallow <think>/<tool_call>, which are real content. */+ if(!T.id_special[101]){ fprintf(stderr," FAIL <|user|> not marked special\n"); fail=1; }+ if(!T.id_special[104]){ fprintf(stderr," FAIL <sop> not marked special\n"); fail=1; }+ if(T.id_special[110]){ fprintf(stderr," FAIL <think> wrongly marked special\n"); fail=1; }+ if(T.id_special[111]){ fprintf(stderr," FAIL <tool_call> wrongly marked special\n"); fail=1; }+ if(!fail) printf(" tokenizer: special flag parsed, <think>/<tool_call> excluded ok\n");++ /* 2. config.json mutilated to one eos, generation_config.json intact:+ * the union must recover all three. This is @woolcoxm's case. */+ { Cfg c; memset(&c,0,sizeof c);+ write_cfg(dir,"config.json","[100]");+ write_cfg(dir,"generation_config.json","[100,101,102]");+ load_cfg(&c,dir);+ if(c.n_stop!=3){ fprintf(stderr," FAIL union: expected 3 stops, got %d\n",c.n_stop); fail=1; }+ else printf(" config eos=[100] + generation_config=[100,101,102] -> 3 stops ok\n"); }++ /* 3. no generation_config.json at all: must not crash, config alone wins */+ { Cfg c; memset(&c,0,sizeof c);+ write_cfg(dir,"config.json","[100,101,102]");+ rm_file(dir,"generation_config.json");+ load_cfg(&c,dir);+ if(c.n_stop!=3){ fprintf(stderr," FAIL no-genconfig: expected 3, got %d\n",c.n_stop); fail=1; }+ else printf(" generation_config.json absent -> config alone, no crash ok\n"); }++ /* 4. BOTH configs mutilated: the tokenizer's special set must still stop the+ * turn on every control token, and must still leave <think> alone. */+ { Cfg c; memset(&c,0,sizeof c);+ write_cfg(dir,"config.json","[100]");+ write_cfg(dir,"generation_config.json","[100]");+ load_cfg(&c,dir);+ stops_arm_tok(&c,-1,&T);+ fail|=expect("endoftext (config)", 100,1);+ fail|=expect("<|user|> (tokenizer)", 101,1);+ fail|=expect("<|observation|>", 102,1);+ fail|=expect("<|assistant|>", 103,1);+ fail|=expect("<sop>", 104,1);+ fail|=expect("<think> must NOT stop",110,0);+ fail|=expect("<tool_call> must NOT stop",111,0);+ fail|=expect("plain token must NOT stop",2,0);+ if(!fail) printf(" both configs mutilated -> tokenizer still stops all 5 control tokens ok\n"); }++ /* 5. T=NULL (validation/oracle path): config stops only, unchanged behaviour */+ { Cfg c; memset(&c,0,sizeof c);+ write_cfg(dir,"config.json","[100]");+ rm_file(dir,"generation_config.json");+ load_cfg(&c,dir);+ stops_arm_tok(&c,-1,NULL);+ fail|=expect("T=NULL: config stop",100,1);+ fail|=expect("T=NULL: no tokenizer sweep",101,0);+ if(!fail) printf(" T=NULL -> config stops only (validation path untouched) ok\n"); }++ rm_file(dir,"config.json"); rm_file(dir,"generation_config.json"); rm_file(dir,"tokenizer.json");+ rmdir(dir);+ if(fail){ printf("test_stops: FAIL\n"); return 1; }+ printf("test_stops: ok\n");+ return 0;+}diff --git a/c/tok.h b/c/tok.hindex f61011e0..7d1140bb 100644--- a/c/tok.h+++ b/c/tok.h@@ -42,6 +42,11 @@ typedef struct {hmap vocab; /* stringa byte-level -> id */hmap merges; /* "left\0right" -> rank */char **id2str; int *id_added; int n_ids; /* id -> stringa; id_added=1 se added-token (output letterale) */+ int *id_special; /* 1 = added-token con "special":true nel tokenizer:+ * token di CONTROLLO (<|user|>, <|assistant|>, <sop>, ...),+ * mai contenuto legittimo di una risposta. Distinto da+ * id_added, che copre anche <think>/<tool_call> ("special"+ * false), i quali sono testo vero e vanno renderizzati. */Special *sp; int nsp; /* added tokens, ordinati per lunghezza decrescente */uint32_t byte2cp[256]; int byte2cp_len[256]; char byte2str[256][3];int16_t cp2byte[1024];@@ -106,6 +111,7 @@ static void tok_load(Tok *T, const char *path){T->n_ids=maxid+1;T->id2str=calloc(T->n_ids,sizeof(char*));T->id_added=calloc(T->n_ids,sizeof(int));+ T->id_special=calloc(T->n_ids,sizeof(int));/* vocab: stringa -> id (capacita' potenza di 2, ~2-3x) */int vc=1; while(vc < vocab->len*2) vc<<=1;@@ -133,6 +139,8 @@ static void tok_load(Tok *T, const char *path){char *content=json_get(a,"content")->str; int id=(int)json_get(a,"id")->num;T->sp[i].str=content; T->sp[i].len=(int)strlen(content); T->sp[i].id=id;T->id2str[id]=content; T->id_added[id]=1;+ jval *sf=json_get(a,"special"); /* "special": true/false */+ if(sf && sf->t==J_BOOL && sf->boolean) T->id_special[id]=1;}qsort(T->sp,T->nsp,sizeof(Special),cmp_sp_len); /* match piu' lungo per primo */}