Skip to content

Run frontier MoE models on hardware you already own — pure C, zero deps, experts streamed from disk. Tiny engine, immense model. 🐦

C

fix(chat): retain role stop tokens in private mode

ZacharyZcR6ccb65a1a95f
diff --git a/c/sample.h b/c/sample.h
index 1df00ad4..3ca2c09f 100644
--- a/c/sample.h
+++ b/c/sample.h
@@ -116,20 +116,23 @@ static void stops_arm_tok(const Cfg *c, int tok_eos, Tok *T){
int nsp = 0;
if (T) for (int id = 0; id < T->n_ids && g_nstop < 64; id++)
if (T->id_special[id] && !is_stop(id)) { g_stop[g_nstop++] = id; nsp++; }
- /* #401: in serve mode keep ONLY <|endoftext|>. Role markers <|user|>/<|observation|>
- * (config stops + tokenizer special set) are boundaries the Python server owns; as
- * hard stops they cut generation the moment the model opens a <tool_call> block,
- * because int4 argmax noise picks a stop-token ID over the correct '<' token.
+ /* #401: in batched gateway mode keep ONLY <|endoftext|>. Role markers
+ * <|user|>/<|observation|> (config stops + tokenizer special set) are boundaries
+ * the Python server owns; as hard stops they cut generation the moment the model
+ * opens a <tool_call> block, because int4 argmax noise picks a stop-token ID over
+ * the correct '<' token. Private `coli chat` also sets SERVE=1 for the byte
+ * protocol, but has no Python StopFilter, so it must retain the full stop set.
*
* #549: on some quantized containers (notably int4-gs64) an end-of-turn special
* OTHER than <|endoftext|> wins the final-token margin, so serve mode never stops and
* runs into hallucinated turns. COLI_SERVE_ALL_STOPS=1 re-arms the full stop set for
* users who are NOT doing tool calls (or whose client owns the tool boundary), at the
* cost of the #401 tool-call safety. Default off — filter unchanged. */
- if (getenv("SERVE") && tok_eos >= 0 && !getenv("COLI_SERVE_ALL_STOPS")) {
+ if (getenv("SERVE") && getenv("SERVE_BATCH") && atoi(getenv("SERVE_BATCH")) &&
+ tok_eos >= 0 && !getenv("COLI_SERVE_ALL_STOPS")) {
int kept = 0;
for (int i = 0; i < g_nstop; i++) if (g_stop[i] == tok_eos) g_stop[kept++] = g_stop[i];
- if (kept < g_nstop) fprintf(stderr, "[stop] serve mode: filtered %d non-EOS stop tokens (tool-call safety, #401)\n", g_nstop - kept);
+ if (kept < g_nstop) fprintf(stderr, "[stop] batched serve mode: filtered %d non-EOS stop tokens (tool-call safety, #401)\n", g_nstop - kept);
g_nstop = kept; nsp = 0;
}
fprintf(stderr, "[stop] %d stop tokens:", g_nstop);
diff --git a/c/tests/test_stops.c b/c/tests/test_stops.c
index 5e7e9d2a..ad4e7d78 100644
--- a/c/tests/test_stops.c
+++ b/c/tests/test_stops.c
@@ -128,6 +128,28 @@ int main(void){
fail|=expect("T=NULL: no tokenizer sweep",101,0);
if(!fail) printf(" T=NULL -> config stops only (validation path untouched) ok\n"); }
+ /* 6. Private chat uses SERVE=1 for the byte protocol but has no Python
+ * StopFilter. Only the batched gateway may reduce the engine stop set. */
+ { Cfg c; memset(&c,0,sizeof c); c.stop_ids[0]=100; c.n_stop=1;
+ setenv("SERVE","1",1); unsetenv("SERVE_BATCH"); unsetenv("COLI_SERVE_ALL_STOPS");
+ stops_arm_tok(&c,100,&T);
+ fail|=expect("private chat keeps <|user|>",101,1);
+ fail|=expect("private chat keeps <|observation|>",102,1);
+ if(g_nstop!=5){ fprintf(stderr," FAIL private chat: expected 5 stops, got %d\n",g_nstop); fail=1; }
+
+ setenv("SERVE_BATCH","1",1);
+ stops_arm_tok(&c,100,&T);
+ fail|=expect("batched gateway keeps EOS",100,1);
+ fail|=expect("batched gateway filters <|user|>",101,0);
+ if(g_nstop!=1){ fprintf(stderr," FAIL batched gateway: expected 1 stop, got %d\n",g_nstop); fail=1; }
+
+ setenv("COLI_SERVE_ALL_STOPS","1",1);
+ stops_arm_tok(&c,100,&T);
+ fail|=expect("gateway override restores <|user|>",101,1);
+ if(g_nstop!=5){ fprintf(stderr," FAIL gateway override: expected 5 stops, got %d\n",g_nstop); fail=1; }
+ unsetenv("COLI_SERVE_ALL_STOPS"); unsetenv("SERVE_BATCH"); unsetenv("SERVE");
+ if(!fail) printf(" private chat keeps 5 stops; batched gateway filters to EOS ok\n"); }
+
rm_file(dir,"config.json"); rm_file(dir,"generation_config.json"); rm_file(dir,"tokenizer.json");
rmdir(dir);
if(fail){ printf("test_stops: FAIL\n"); return 1; }