Skip to content

Run frontier MoE models on hardware you already own — pure C, zero deps, experts streamed from disk. Tiny engine, immense model. 🐦

C

Merge #192: serve stage 2 — response_format, per-request grammars, grammar-forced drafts

Vincenzo Fornaro880cfb4ec59a
diff --cc c/colibri.c
index 8878ae3a,9a231642..a94f7951
--- a/c/colibri.c
+++ b/c/colibri.c
@@@ -4564,24 -4446,12 +4583,24 @@@ static void run_text(Model *m, const ch
Tok T; tok_load(&T,tkp);
int eos=tok_id_of(&T,"<|endoftext|>");
stops_arm_tok(&m->c, eos, &T);
- grammar_setup(&T); /* metodo F: GRAMMAR=file.gbnf (#48) */
+ grammar_setup(&g_grd,&T); /* metodo F: GRAMMAR=file.gbnf (#48) */
if(g_temp<0) g_temp=0.7f; /* auto: 0.7, NON l'1.0 ufficiale — la coda della
* distribuzione int4 e' rumore di quantizzazione */
- int cap=(int)strlen(prompt)+16; int *pids=malloc(cap*sizeof(int));
+ int cap=(int)strlen(prompt)+16; int *pids=malloc((cap+2)*sizeof(int));
int np=tok_encode(&T,prompt,(int)strlen(prompt),pids,cap);
if(np<1){ fprintf(stderr,"prompt is empty after tokenization\n"); return; }
+ /* GLM prefix (#108): every GLM training sequence starts [gMASK]<sop>; without it the
+ * model is out-of-distribution and generates garbage. Serve/SCORE modes prepend it;
+ * run_text must too. CHAT_TEMPLATE=0 disables (raw prompt, for debugging/non-GLM).
+ * A prompt that already starts with the prefix passes through intact. */
+ int templ=getenv("CHAT_TEMPLATE")?atoi(getenv("CHAT_TEMPLATE")):1;
+ if(templ){
+ int gmask=tok_id_of(&T,"[gMASK]"), sop=tok_id_of(&T,"<sop>");
+ if(gmask>=0 && sop>=0 && (np<2 || pids[0]!=gmask || pids[1]!=sop)){
+ memmove(pids+2,pids,np*sizeof(int));
+ pids[0]=gmask; pids[1]=sop; np+=2;
+ }
+ }
printf("prompt: %d tokens | generating up to %d (EOS stop=%d) | n-gram draft=%d\n", np, ngen, eos, g_draft);
fputs(prompt,stdout); fflush(stdout);
kv_alloc(m, np+ngen+g_draft+2);