Run frontier MoE models on hardware you already own — pure C, zero deps, experts streamed from disk. Tiny engine, immense model. 🐦
C
colibri: add CONSIST=1 prefill/decode self-consistency check
phreakociousde836845d624
diff --git a/c/colibri.c b/c/colibri.cindex 24ab3b3a..d1779fed 100644--- a/c/colibri.c+++ b/c/colibri.c@@ -8213,6 +8213,106 @@ static void run_replay(Model *m, const int *full, int nfull, int np){#endif}+/* CONSIST=1: prefill/decode self-consistency. The same positions are evaluated+ * twice -- arm A pushes the whole sequence through step_all in one batched+ * prefill, arm B prefills the prompt prefix and then walks the continuation one+ * token at a time through the KV cache, exactly as generate() does. The two+ * arms share weights, so any disagreement beyond float accumulation order is a+ * KV-addressing or masking defect in one of them.+ *+ * Unlike TF=1 this needs no oracle file and no reference implementation: it is+ * the engine against itself, so it runs on any model, any quantization and any+ * backend -- including the ones no CI runner has a GPU for. It also covers+ * COLI_PREFILL_CHUNK, which only arm B honours.+ *+ * The gate is the largest RELATIVE logit gap, because that is the quantity that+ * separates the two failure classes: reordered f32 accumulation over the hidden+ * dim lands near D*eps (~1e-3 at D=7168), a wrong mask or a misaddressed KV row+ * lands at O(1). Argmax flips are reported but NOT gated: a flip requires+ * |a[ia]-a[ib]| < 2*gap by construction, so "the flip is explained by the gap"+ * is true for every flip and would be an assertion that cannot fail. */+#define CONSIST_MAX_S 512 /* step_all writes S*D floats into m->h_all, sized 512*D */++static void run_consist(Model *m, const int *full, int nfull, int np){+ Cfg *c=&m->c; int V=c->vocab;+ if(np<2||nfull<=np){ fprintf(stderr,"CONSIST requires a non-empty prompt and continuation\n"); return; }+ if(nfull>CONSIST_MAX_S){+ fprintf(stderr,"CONSIST: %d tokens exceeds the %d-token step_all ceiling\n",nfull,CONSIST_MAX_S); return; }++ int saved_draft=g_draft; g_draft=0; /* mtp_absorb fires in arm B only; keep the arms comparable */++ kv_alloc(m,nfull+2);+ float *A=step_all(m,full,nfull,0); /* arm A: one batched prefill */++ kv_alloc(m,nfull+2); /* arm B: prefix, then one token at a time */+ float *lo=step(m,full,np-1,0); free(lo);++ double worst=0, worst_abs=0; int worst_pos=-1, flips=0, compared=0;+ double flip_margin=0; int flip_pos=-1;+ for(int i=np-1;i<nfull-1;i++){+ lo=step(m,full+i,1,i);+ const float *a=A+(int64_t)i*V;+ double gap=0, scale=0; int ia=0, ib=0;+ for(int v=0;v<V;v++){+ double d=fabs((double)a[v]-(double)lo[v]); if(d>gap) gap=d;+ double s=fabs((double)a[v]); if(s>scale) scale=s;+ if(a[v]>a[ia]) ia=v;+ if(lo[v]>lo[ib]) ib=v;+ }+ double rel = scale>0 ? gap/scale : gap;+ if(rel>worst){ worst=rel; worst_abs=gap; worst_pos=i; }+ if(ia!=ib){ flips++;+ double margin=fabs((double)a[ia]-(double)a[ib]);+ if(margin>flip_margin){ flip_margin=margin; flip_pos=i; } }+ compared++;+ free(lo);+ }+ free(A);+ g_draft=saved_draft;++ double tol = getenv("CONSIST_TOL") ? atof(getenv("CONSIST_TOL")) : 1e-2;+ printf("CONSIST prefill vs decode: %d positions | worst relative gap %.3e (abs %.3e) at pos %d | tol %.1e\n",+ compared, worst, worst_abs, worst_pos, tol);+ if(flips) printf("CONSIST argmax flips: %d/%d | widest top1-top2 margin %.3e at pos %d (near-ties: informational)\n",+ flips, compared, flip_margin, flip_pos);+ if(worst>tol){+ fprintf(stderr,"CONSIST FAIL: worst relative gap %.3e exceeds tol %.1e — "+ "prefill and decode do not agree on the same positions\n", worst, tol);+ exit(1);+ }+ printf("CONSIST OK\n");+}++/* CONSIST driven by PROMPT: the prompt's own tokens supply both arms, so the check+ * needs no ref file at all and runs against any model the engine can load. The split+ * point is how much of it arm B prefills before stepping the rest one token at a time;+ * CONSIST_NP overrides the default halfway split. */+static void run_consist_prompt(Model *m, const char *snap, const char *prompt){+ char tkp[2048]; snprintf(tkp,sizeof(tkp),"%s/tokenizer.json",snap);+ Tok T; tok_load(&T,tkp);+ int cap=(int)strlen(prompt)+16; int *ids=malloc(((size_t)cap+4)*sizeof(int));+ if(!ids){ fprintf(stderr,"CONSIST: out of memory\n"); tok_free(&T); return; }+ int n=tok_encode(&T,prompt,(int)strlen(prompt),ids,cap);+ if(n<1){ fprintf(stderr,"CONSIST: prompt is empty after tokenization\n"); free(ids); tok_free(&T); return; }+ /* The same GLM prefix run_text applies (#108). Without [gMASK]<sop> the sequence is+ * out-of-distribution; both arms would then agree, but on garbage. */+ int templ=getenv("CHAT_TEMPLATE")?atoi(getenv("CHAT_TEMPLATE")):1;+ if(templ){+ int gmask=tok_id_of(&T,"[gMASK]"), sop=tok_id_of(&T,"<sop>");+ if(gmask>=0 && sop>=0 && (n<2 || ids[0]!=gmask || ids[1]!=sop)){+ memmove(ids+2,ids,(size_t)n*sizeof(int)); ids[0]=gmask; ids[1]=sop; n+=2;+ }+ }+ tok_free(&T); /* ids is self-contained from here (tok.h pairs load/free) */+ int np = getenv("CONSIST_NP") ? atoi(getenv("CONSIST_NP")) : n/2;+ if(np<2) np=2;+ if(np>=n){ fprintf(stderr,"CONSIST: prefix %d leaves no continuation in %d tokens\n",np,n);+ free(ids); return; }+ printf("CONSIST from prompt: %d tokens | prefix %d | continuation %d\n", n, np, n-np);+ run_consist(m, ids, n, np);+ free(ids);+}+/* generazione reale: tokenizza PROMPT, prefill + decode greedy con stop su EOS,* detokenizza e stampa il testo in streaming. */static void run_text(Model *m, const char *snap, const char *prompt, int ngen){@@ -11513,6 +11613,14 @@ int main(int argc, char **argv){/* modo testo reale: PROMPT="..." [NGEN=n] -> tokenizza, genera, detokenizza */const char *user_prompt = coli_user_prompt(); /* ignores cmd.exe's PROMPT template (#271) */+ /* CONSIST with a PROMPT takes its tokens from the prompt, so it never reaches the+ * oracle path below and needs no ref file. */+ if(user_prompt && getenv("CONSIST")){+ run_consist_prompt(&m, snap, user_prompt);+ if(stats) stats_dump(&m,stats);+ return 0;+ }+if(user_prompt){int ngen=getenv("NGEN")?atoi(getenv("NGEN")):64;run_text(&m, snap, user_prompt, ngen);@@ -11553,6 +11661,12 @@ int main(int argc, char **argv){return 0;}+ if(getenv("CONSIST")){+ run_consist(&m,full,nfull,np);+ if(stats) stats_dump(&m,stats);+ return 0;+ }+if(getenv("TF")){int *tf=read_arr(ref,"tf_pred",&(int){0});int *pred=malloc(nfull*sizeof(int)); double tt=now_s();