Run frontier MoE models on hardware you already own — pure C, zero deps, experts streamed from disk. Tiny engine, immense model. 🐦
C
fix(chat): retain role stop tokens in private mode
ZacharyZcR6ccb65a1a95f
diff --git a/c/sample.h b/c/sample.hindex 1df00ad4..3ca2c09f 100644--- a/c/sample.h+++ b/c/sample.h@@ -116,20 +116,23 @@ static void stops_arm_tok(const Cfg *c, int tok_eos, Tok *T){int nsp = 0;if (T) for (int id = 0; id < T->n_ids && g_nstop < 64; id++)if (T->id_special[id] && !is_stop(id)) { g_stop[g_nstop++] = id; nsp++; }- /* #401: in serve mode keep ONLY <|endoftext|>. Role markers <|user|>/<|observation|>- * (config stops + tokenizer special set) are boundaries the Python server owns; as- * hard stops they cut generation the moment the model opens a <tool_call> block,- * because int4 argmax noise picks a stop-token ID over the correct '<' token.+ /* #401: in batched gateway mode keep ONLY <|endoftext|>. Role markers+ * <|user|>/<|observation|> (config stops + tokenizer special set) are boundaries+ * the Python server owns; as hard stops they cut generation the moment the model+ * opens a <tool_call> block, because int4 argmax noise picks a stop-token ID over+ * the correct '<' token. Private `coli chat` also sets SERVE=1 for the byte+ * protocol, but has no Python StopFilter, so it must retain the full stop set.** #549: on some quantized containers (notably int4-gs64) an end-of-turn special* OTHER than <|endoftext|> wins the final-token margin, so serve mode never stops and* runs into hallucinated turns. COLI_SERVE_ALL_STOPS=1 re-arms the full stop set for* users who are NOT doing tool calls (or whose client owns the tool boundary), at the* cost of the #401 tool-call safety. Default off — filter unchanged. */- if (getenv("SERVE") && tok_eos >= 0 && !getenv("COLI_SERVE_ALL_STOPS")) {+ if (getenv("SERVE") && getenv("SERVE_BATCH") && atoi(getenv("SERVE_BATCH")) &&+ tok_eos >= 0 && !getenv("COLI_SERVE_ALL_STOPS")) {int kept = 0;for (int i = 0; i < g_nstop; i++) if (g_stop[i] == tok_eos) g_stop[kept++] = g_stop[i];- if (kept < g_nstop) fprintf(stderr, "[stop] serve mode: filtered %d non-EOS stop tokens (tool-call safety, #401)\n", g_nstop - kept);+ if (kept < g_nstop) fprintf(stderr, "[stop] batched serve mode: filtered %d non-EOS stop tokens (tool-call safety, #401)\n", g_nstop - kept);g_nstop = kept; nsp = 0;}fprintf(stderr, "[stop] %d stop tokens:", g_nstop);diff --git a/c/tests/test_stops.c b/c/tests/test_stops.cindex 5e7e9d2a..ad4e7d78 100644--- a/c/tests/test_stops.c+++ b/c/tests/test_stops.c@@ -128,6 +128,28 @@ int main(void){fail|=expect("T=NULL: no tokenizer sweep",101,0);if(!fail) printf(" T=NULL -> config stops only (validation path untouched) ok\n"); }+ /* 6. Private chat uses SERVE=1 for the byte protocol but has no Python+ * StopFilter. Only the batched gateway may reduce the engine stop set. */+ { Cfg c; memset(&c,0,sizeof c); c.stop_ids[0]=100; c.n_stop=1;+ setenv("SERVE","1",1); unsetenv("SERVE_BATCH"); unsetenv("COLI_SERVE_ALL_STOPS");+ stops_arm_tok(&c,100,&T);+ fail|=expect("private chat keeps <|user|>",101,1);+ fail|=expect("private chat keeps <|observation|>",102,1);+ if(g_nstop!=5){ fprintf(stderr," FAIL private chat: expected 5 stops, got %d\n",g_nstop); fail=1; }++ setenv("SERVE_BATCH","1",1);+ stops_arm_tok(&c,100,&T);+ fail|=expect("batched gateway keeps EOS",100,1);+ fail|=expect("batched gateway filters <|user|>",101,0);+ if(g_nstop!=1){ fprintf(stderr," FAIL batched gateway: expected 1 stop, got %d\n",g_nstop); fail=1; }++ setenv("COLI_SERVE_ALL_STOPS","1",1);+ stops_arm_tok(&c,100,&T);+ fail|=expect("gateway override restores <|user|>",101,1);+ if(g_nstop!=5){ fprintf(stderr," FAIL gateway override: expected 5 stops, got %d\n",g_nstop); fail=1; }+ unsetenv("COLI_SERVE_ALL_STOPS"); unsetenv("SERVE_BATCH"); unsetenv("SERVE");+ if(!fail) printf(" private chat keeps 5 stops; batched gateway filters to EOS ok\n"); }+rm_file(dir,"config.json"); rm_file(dir,"generation_config.json"); rm_file(dir,"tokenizer.json");rmdir(dir);if(fail){ printf("test_stops: FAIL\n"); return 1; }