Run frontier MoE models on hardware you already own — pure C, zero deps, experts streamed from disk. Tiny engine, immense model. 🐦
C
fix(engine): filter non-EOS stop tokens in serve mode to prevent premature halt on tool calls
Colibri Developer26bd8b403a8d
diff --git a/c/glm.c b/c/glm.cindex 00c70768..3237c0a4 100644--- a/c/glm.c+++ b/c/glm.c@@ -4115,6 +4115,19 @@ static void stops_arm(const Cfg *c, int tok_eos){g_nstop=0;for(int i=0;i<c->n_stop;i++) g_stop[g_nstop++]=c->stop_ids[i];if(tok_eos>=0 && !is_stop(tok_eos)) g_stop[g_nstop++]=tok_eos;+ /* In serve mode (API), keep only <|endoftext|> as a stop token. The tokens+ * <|user|> and <|observation|> are role markers that the Python server+ * handles — keeping them as stop tokens causes the engine to halt+ * prematurely when the model tries to emit <tool_call> blocks, because+ * int4-quantized logits can be noisy enough that argmax picks a stop-token+ * ID instead of the correct tool-call token. (#401) */+ if(getenv("SERVE")){+ int kept=0;+ for(int i=0;i<g_nstop;i++) if(g_stop[i]==tok_eos) g_stop[kept++]=g_stop[i];+ if(kept<g_nstop) fprintf(stderr,"[stop] serve mode: filtered %d non-EOS stop tokens (tool-call safety, #401)\n",+ g_nstop-kept);+ g_nstop=kept;+ }fprintf(stderr,"[stop] %d stop tokens:",g_nstop);for(int i=0;i<g_nstop;i++) fprintf(stderr," %d",g_stop[i]);fprintf(stderr,"\n");