Skip to content

Run frontier MoE models on hardware you already own — pure C, zero deps, experts streamed from disk. Tiny engine, immense model. 🐦

C

colibri: add CONSIST=1 prefill/decode self-consistency check

phreakociousde836845d624
diff --git a/c/colibri.c b/c/colibri.c
index 24ab3b3a..d1779fed 100644
--- a/c/colibri.c
+++ b/c/colibri.c
@@ -8213,6 +8213,106 @@ static void run_replay(Model *m, const int *full, int nfull, int np){
#endif
}
+/* CONSIST=1: prefill/decode self-consistency. The same positions are evaluated
+ * twice -- arm A pushes the whole sequence through step_all in one batched
+ * prefill, arm B prefills the prompt prefix and then walks the continuation one
+ * token at a time through the KV cache, exactly as generate() does. The two
+ * arms share weights, so any disagreement beyond float accumulation order is a
+ * KV-addressing or masking defect in one of them.
+ *
+ * Unlike TF=1 this needs no oracle file and no reference implementation: it is
+ * the engine against itself, so it runs on any model, any quantization and any
+ * backend -- including the ones no CI runner has a GPU for. It also covers
+ * COLI_PREFILL_CHUNK, which only arm B honours.
+ *
+ * The gate is the largest RELATIVE logit gap, because that is the quantity that
+ * separates the two failure classes: reordered f32 accumulation over the hidden
+ * dim lands near D*eps (~1e-3 at D=7168), a wrong mask or a misaddressed KV row
+ * lands at O(1). Argmax flips are reported but NOT gated: a flip requires
+ * |a[ia]-a[ib]| < 2*gap by construction, so "the flip is explained by the gap"
+ * is true for every flip and would be an assertion that cannot fail. */
+#define CONSIST_MAX_S 512 /* step_all writes S*D floats into m->h_all, sized 512*D */
+
+static void run_consist(Model *m, const int *full, int nfull, int np){
+ Cfg *c=&m->c; int V=c->vocab;
+ if(np<2||nfull<=np){ fprintf(stderr,"CONSIST requires a non-empty prompt and continuation\n"); return; }
+ if(nfull>CONSIST_MAX_S){
+ fprintf(stderr,"CONSIST: %d tokens exceeds the %d-token step_all ceiling\n",nfull,CONSIST_MAX_S); return; }
+
+ int saved_draft=g_draft; g_draft=0; /* mtp_absorb fires in arm B only; keep the arms comparable */
+
+ kv_alloc(m,nfull+2);
+ float *A=step_all(m,full,nfull,0); /* arm A: one batched prefill */
+
+ kv_alloc(m,nfull+2); /* arm B: prefix, then one token at a time */
+ float *lo=step(m,full,np-1,0); free(lo);
+
+ double worst=0, worst_abs=0; int worst_pos=-1, flips=0, compared=0;
+ double flip_margin=0; int flip_pos=-1;
+ for(int i=np-1;i<nfull-1;i++){
+ lo=step(m,full+i,1,i);
+ const float *a=A+(int64_t)i*V;
+ double gap=0, scale=0; int ia=0, ib=0;
+ for(int v=0;v<V;v++){
+ double d=fabs((double)a[v]-(double)lo[v]); if(d>gap) gap=d;
+ double s=fabs((double)a[v]); if(s>scale) scale=s;
+ if(a[v]>a[ia]) ia=v;
+ if(lo[v]>lo[ib]) ib=v;
+ }
+ double rel = scale>0 ? gap/scale : gap;
+ if(rel>worst){ worst=rel; worst_abs=gap; worst_pos=i; }
+ if(ia!=ib){ flips++;
+ double margin=fabs((double)a[ia]-(double)a[ib]);
+ if(margin>flip_margin){ flip_margin=margin; flip_pos=i; } }
+ compared++;
+ free(lo);
+ }
+ free(A);
+ g_draft=saved_draft;
+
+ double tol = getenv("CONSIST_TOL") ? atof(getenv("CONSIST_TOL")) : 1e-2;
+ printf("CONSIST prefill vs decode: %d positions | worst relative gap %.3e (abs %.3e) at pos %d | tol %.1e\n",
+ compared, worst, worst_abs, worst_pos, tol);
+ if(flips) printf("CONSIST argmax flips: %d/%d | widest top1-top2 margin %.3e at pos %d (near-ties: informational)\n",
+ flips, compared, flip_margin, flip_pos);
+ if(worst>tol){
+ fprintf(stderr,"CONSIST FAIL: worst relative gap %.3e exceeds tol %.1e — "
+ "prefill and decode do not agree on the same positions\n", worst, tol);
+ exit(1);
+ }
+ printf("CONSIST OK\n");
+}
+
+/* CONSIST driven by PROMPT: the prompt's own tokens supply both arms, so the check
+ * needs no ref file at all and runs against any model the engine can load. The split
+ * point is how much of it arm B prefills before stepping the rest one token at a time;
+ * CONSIST_NP overrides the default halfway split. */
+static void run_consist_prompt(Model *m, const char *snap, const char *prompt){
+ char tkp[2048]; snprintf(tkp,sizeof(tkp),"%s/tokenizer.json",snap);
+ Tok T; tok_load(&T,tkp);
+ int cap=(int)strlen(prompt)+16; int *ids=malloc(((size_t)cap+4)*sizeof(int));
+ if(!ids){ fprintf(stderr,"CONSIST: out of memory\n"); tok_free(&T); return; }
+ int n=tok_encode(&T,prompt,(int)strlen(prompt),ids,cap);
+ if(n<1){ fprintf(stderr,"CONSIST: prompt is empty after tokenization\n"); free(ids); tok_free(&T); return; }
+ /* The same GLM prefix run_text applies (#108). Without [gMASK]<sop> the sequence is
+ * out-of-distribution; both arms would then agree, but on garbage. */
+ int templ=getenv("CHAT_TEMPLATE")?atoi(getenv("CHAT_TEMPLATE")):1;
+ if(templ){
+ int gmask=tok_id_of(&T,"[gMASK]"), sop=tok_id_of(&T,"<sop>");
+ if(gmask>=0 && sop>=0 && (n<2 || ids[0]!=gmask || ids[1]!=sop)){
+ memmove(ids+2,ids,(size_t)n*sizeof(int)); ids[0]=gmask; ids[1]=sop; n+=2;
+ }
+ }
+ tok_free(&T); /* ids is self-contained from here (tok.h pairs load/free) */
+ int np = getenv("CONSIST_NP") ? atoi(getenv("CONSIST_NP")) : n/2;
+ if(np<2) np=2;
+ if(np>=n){ fprintf(stderr,"CONSIST: prefix %d leaves no continuation in %d tokens\n",np,n);
+ free(ids); return; }
+ printf("CONSIST from prompt: %d tokens | prefix %d | continuation %d\n", n, np, n-np);
+ run_consist(m, ids, n, np);
+ free(ids);
+}
+
/* generazione reale: tokenizza PROMPT, prefill + decode greedy con stop su EOS,
* detokenizza e stampa il testo in streaming. */
static void run_text(Model *m, const char *snap, const char *prompt, int ngen){
@@ -11513,6 +11613,14 @@ int main(int argc, char **argv){
/* modo testo reale: PROMPT="..." [NGEN=n] -> tokenizza, genera, detokenizza */
const char *user_prompt = coli_user_prompt(); /* ignores cmd.exe's PROMPT template (#271) */
+ /* CONSIST with a PROMPT takes its tokens from the prompt, so it never reaches the
+ * oracle path below and needs no ref file. */
+ if(user_prompt && getenv("CONSIST")){
+ run_consist_prompt(&m, snap, user_prompt);
+ if(stats) stats_dump(&m,stats);
+ return 0;
+ }
+
if(user_prompt){
int ngen=getenv("NGEN")?atoi(getenv("NGEN")):64;
run_text(&m, snap, user_prompt, ngen);
@@ -11553,6 +11661,12 @@ int main(int argc, char **argv){
return 0;
}
+ if(getenv("CONSIST")){
+ run_consist(&m,full,nfull,np);
+ if(stats) stats_dump(&m,stats);
+ return 0;
+ }
+
if(getenv("TF")){
int *tf=read_arr(ref,"tf_pred",&(int){0});
int *pred=malloc(nfull*sizeof(int)); double tt=now_s();