Run frontier MoE models on hardware you already own — pure C, zero deps, experts streamed from disk. Tiny engine, immense model. 🐦
C
Merge #192: serve stage 2 — response_format, per-request grammars, grammar-forced drafts
Vincenzo Fornaro880cfb4ec59a
diff --cc c/colibri.cindex 8878ae3a,9a231642..a94f7951--- a/c/colibri.c+++ b/c/colibri.c@@@ -4564,24 -4446,12 +4583,24 @@@ static void run_text(Model *m, const chTok T; tok_load(&T,tkp);int eos=tok_id_of(&T,"<|endoftext|>");stops_arm_tok(&m->c, eos, &T);- grammar_setup(&T); /* metodo F: GRAMMAR=file.gbnf (#48) */+ grammar_setup(&g_grd,&T); /* metodo F: GRAMMAR=file.gbnf (#48) */if(g_temp<0) g_temp=0.7f; /* auto: 0.7, NON l'1.0 ufficiale — la coda della* distribuzione int4 e' rumore di quantizzazione */- int cap=(int)strlen(prompt)+16; int *pids=malloc(cap*sizeof(int));+ int cap=(int)strlen(prompt)+16; int *pids=malloc((cap+2)*sizeof(int));int np=tok_encode(&T,prompt,(int)strlen(prompt),pids,cap);if(np<1){ fprintf(stderr,"prompt is empty after tokenization\n"); return; }+ /* GLM prefix (#108): every GLM training sequence starts [gMASK]<sop>; without it the+ * model is out-of-distribution and generates garbage. Serve/SCORE modes prepend it;+ * run_text must too. CHAT_TEMPLATE=0 disables (raw prompt, for debugging/non-GLM).+ * A prompt that already starts with the prefix passes through intact. */+ int templ=getenv("CHAT_TEMPLATE")?atoi(getenv("CHAT_TEMPLATE")):1;+ if(templ){+ int gmask=tok_id_of(&T,"[gMASK]"), sop=tok_id_of(&T,"<sop>");+ if(gmask>=0 && sop>=0 && (np<2 || pids[0]!=gmask || pids[1]!=sop)){+ memmove(pids+2,pids,np*sizeof(int));+ pids[0]=gmask; pids[1]=sop; np+=2;+ }+ }printf("prompt: %d tokens | generating up to %d (EOS stop=%d) | n-gram draft=%d\n", np, ngen, eos, g_draft);fputs(prompt,stdout); fflush(stdout);kv_alloc(m, np+ngen+g_draft+2);