Run frontier MoE models on hardware you already own β pure C, zero deps, experts streamed from disk. Tiny engine, immense model. π¦
C
feat(spec): frozen-corpus draft source (COLI_DRAFT_CORPUS)
terrizoaguimorda7e972bf0ce
diff --git a/c/Makefile b/c/Makefileindex 71b4fd30..1e0fbae5 100644--- a/c/Makefile+++ b/c/Makefile@@ -214,7 +214,7 @@ elsePYTHON ?= python3endifCUDA_OBJ =-TEST_BINS = tests/test_json$(EXE) tests/test_st$(EXE) tests/test_st_mirror$(EXE) tests/test_st_pread$(EXE) tests/test_tier$(EXE) tests/test_grammar$(EXE) tests/test_schema_gbnf$(EXE) tests/test_decode_batch$(EXE) tests/test_idot$(EXE) tests/test_i4_grouped$(EXE) tests/test_stops$(EXE) tests/test_topp$(EXE) tests/test_sample_nan$(EXE) tests/test_temp_env$(EXE) tests/test_tok_o200k$(EXE) tests/test_kv_alloc$(EXE) tests/test_int3$(EXE) tests/test_int3_load$(EXE) tests/test_i4_acc512$(EXE) tests/test_compat_direct$(EXE) tests/test_dsa_select$(EXE) tests/test_logit_nan$(EXE) tests/test_router_nan$(EXE) tests/test_pipe_block$(EXE) tests/test_e8_kernel$(EXE) tests/test_rans$(EXE)+TEST_BINS = tests/test_json$(EXE) tests/test_st$(EXE) tests/test_st_mirror$(EXE) tests/test_st_pread$(EXE) tests/test_tier$(EXE) tests/test_grammar$(EXE) tests/test_schema_gbnf$(EXE) tests/test_decode_batch$(EXE) tests/test_idot$(EXE) tests/test_i4_grouped$(EXE) tests/test_stops$(EXE) tests/test_topp$(EXE) tests/test_sample_nan$(EXE) tests/test_temp_env$(EXE) tests/test_tok_o200k$(EXE) tests/test_kv_alloc$(EXE) tests/test_int3$(EXE) tests/test_int3_load$(EXE) tests/test_i4_acc512$(EXE) tests/test_compat_direct$(EXE) tests/test_dsa_select$(EXE) tests/test_logit_nan$(EXE) tests/test_router_nan$(EXE) tests/test_pipe_block$(EXE) tests/test_e8_kernel$(EXE) tests/test_rans$(EXE) tests/test_corpus_draft$(EXE)ifneq (,$(LINUX))TEST_BINS += tests/test_uring$(EXE)endif@@ -525,6 +525,9 @@ tests/test_compat_direct$(EXE): tests/test_compat_direct.c compat.htests/test_dsa_select$(EXE): tests/test_dsa_select.c colibri.c st.h uring.h json.h tok.h tok_unicode.h compat.h grammar.h tier.h quant.h sample.h kv_persist.h telemetry.h$(CC) $(CFLAGS) $< -o $@ $(LDFLAGS)+tests/test_corpus_draft$(EXE): tests/test_corpus_draft.c colibri.c st.h uring.h json.h tok.h tok_unicode.h compat.h grammar.h tier.h quant.h sample.h kv_persist.h telemetry.h+ $(CC) $(CFLAGS) $< -o $@ $(LDFLAGS)+# bench_dsa_select is a microbenchmark (old qsort vs new quickselect partial-select, #356),# NOT a test gate -- intentionally absent from TEST_BINS. Build on demand: make tests/bench_dsa_selecttests/bench_dsa_select$(EXE): tests/bench_dsa_select.c colibri.c st.h uring.h json.h tok.h tok_unicode.h compat.h grammar.h tier.h quant.h sample.h kv_persist.h telemetry.hdiff --git a/c/colibri.c b/c/colibri.cindex 2e493019..badb3446 100644--- a/c/colibri.c+++ b/c/colibri.c@@ -4524,6 +4524,53 @@ static float *step_decode_batch(Model *m, const DecodeRow *rows, int S){/* METODO E β prompt-lookup: cerca l'occorrenza piu' recente dell'ultimo bigramma nel* contesto e propone i token che la seguirono. Zero pesi extra, zero costo: e' solo* un'ipotesi che il modello verifichera'. */+/* ---- CORPUS DRAFTS (REST-style external draft source) ----------------------+ * COLI_DRAFT_CORPUS=<file> whitespace-separated token ids of past generations+ * (build one with TOKENS=1, which already dumps them)+ * COLI_CORPUS_K=n proposal depth, default 8 (cap 48: batch[64] in spec_decode)+ * Proposals come from the longest suffix of the live context that also occurs in+ * the corpus; the engine's existing verification accepts only what it would have+ * generated anyway, so output stays byte-identical (lossless by construction).+ * Unlike MTP (1 token/forward) a corpus hit proposes a whole span at once. */+static int *g_corp=NULL; static long g_corp_n=0; static int g_corp_k=0;+static uint64_t g_corp_prop=0, g_corp_acc=0;+static void corpus_load(void){+ const char *p=getenv("COLI_DRAFT_CORPUS");+ if(!p||!*p) return;+ FILE *f=fopen(p,"rb");+ if(!f){ fprintf(stderr,"[CORPUS] cannot open %s\n",p); return; }+ long cap=1<<16; g_corp=malloc((size_t)cap*sizeof(int)); g_corp_n=0;+ int v;+ while(g_corp && fscanf(f,"%d",&v)==1){+ if(g_corp_n>=cap){ cap*=2; int *n2=realloc(g_corp,(size_t)cap*sizeof(int));+ if(!n2){ free(g_corp); g_corp=NULL; break; } g_corp=n2; }+ g_corp[g_corp_n++]=v;+ }+ fclose(f);+ if(!g_corp){ g_corp_n=0; fprintf(stderr,"[CORPUS] OOM\n"); return; }+ g_corp_k=getenv("COLI_CORPUS_K")?atoi(getenv("COLI_CORPUS_K")):8;+ if(g_corp_k<1) g_corp_k=1;+ if(g_corp_k>48) g_corp_k=48;+ fprintf(stderr,"[CORPUS] %ld ids from %s (draft depth %d)\n",g_corp_n,p,g_corp_k);+}+static int corpus_draft(const int *ctx, int nctx, int *out, int k, int minn, int maxn){+ /* il minimo utile e' un match di `minn` piu' ALMENO un token da proporre:+ * sotto quella soglia non esiste proposta possibile (#test_corpus_draft) */+ if(!g_corp||g_corp_n<(long)minn+1||nctx<minn||k<1) return 0;+ for(int n=maxn;n>=minn;n--){+ if(n>nctx) continue;+ const int *suf=ctx+nctx-n;+ for(long i=g_corp_n-n-1;i>=0;i--){+ int ok=1;+ for(int j=0;j<n;j++) if(g_corp[i+j]!=suf[j]||g_corp[i+j]<0){ ok=0; break; }+ if(!ok) continue;+ int g=0;+ for(int j=0;j<k && i+n+j<g_corp_n && g_corp[i+n+j]>=0;j++) out[g++]=g_corp[i+n+j];+ if(g>0) return g;+ }+ }+ return 0;+}static int ngram_draft(const int *ids, int len, int G, int *draft){if(len<4 || G<1) return 0;int a=ids[len-2], b=ids[len-1];@@ -4758,6 +4805,7 @@ static int spec_decode(Model *m, int *all, int kv, int n_new, int eos, float *lo* permanent latch β a transient collapse no longer kills MTP for the whole session. */enum { GUARD_PAUSE_TOKENS = 256 };uint64_t gd_prop0=m->mtp_prop, gd_acc0=m->mtp_acc; int gd_pause=0;+ uint64_t cp_prop0=g_corp_prop, cp_acc0=g_corp_acc; int cp_pause=0;while(emitted<n_new && !done && !g_intr && !g_mux_stop && !g_mux_cancel){/* g_intr / g_mux_*: stessa uscita del tetto n_new (#678) */int next=pick_tok(logit,V,carry_ban); carry_ban=-1; free(logit); logit=NULL;@@ -4771,6 +4819,31 @@ static int spec_decode(Model *m, int *all, int kv, int n_new, int eos, float *log=grammar_draft(&g_grd,draft,g_grd.max);if(g>0) gsrc=1;}+ if(!g && g_corp && g_corp_k>0){ /* corpus: uno span intero per forward+ * dove il contesto ricalca il congelato */+ /* Guard di accettazione (stessa forma di quello MTP sopra, soglia diversa).+ * Un draft di corpus RIFIUTATO costa piu' di uno MTP: la verifica batcha+ * 1+K righe e in un MoE ogni riga attiva i propri expert, quindi lo spreco+ * scala con la profondita'. Misurato su GLM-5.2/H200: 90% acceptance =+ * +22% decode, 19% = -25% (il break-even sta circa a meta'). Sotto+ * COLI_CORPUS_MINACC (default 50%) la fonte si mette in pausa. */+ if(cp_pause>0){ cp_pause--; if(!cp_pause){ cp_prop0=g_corp_prop; cp_acc0=g_corp_acc; } }+ else {+ uint64_t pw=g_corp_prop-cp_prop0, aw=g_corp_acc-cp_acc0;+ int minacc=getenv("COLI_CORPUS_MINACC")?atoi(getenv("COLI_CORPUS_MINACC")):50;+ if(minacc<0) minacc=0; if(minacc>100) minacc=100;+ if(pw>=24 && aw*100 < pw*(uint64_t)minacc){+ fprintf(stderr,"[CORPUS] %.0f%% acceptance over the last %llu proposals (<%d%%): "+ "drafts paused for %d tokens\n",+ 100.0*aw/pw,(unsigned long long)pw,minacc,(int)GUARD_PAUSE_TOKENS);+ cp_pause=GUARD_PAUSE_TOKENS;+ }+ }+ if(!cp_pause){+ g=corpus_draft(all,kv+1,draft,g_corp_k,3,8);+ if(g>0){ gsrc=3; g_corp_prop+=(uint64_t)g; }+ }+ }if(!g && g_draft>0 && m->has_mtp){/* pausa adattiva: draft che non vengono mai accettati = solo tassa disco,* ma il vecchio g_draft=0 era permanente. EN: adaptive pause; the old@@ -4810,6 +4883,7 @@ static int spec_decode(Model *m, int *all, int kv, int n_new, int eos, float *lo}if(gsrc==1) g_grd.acc+=(uint64_t)k;else if(gsrc==2 && m->has_mtp) m->mtp_acc+=k;+ else if(gsrc==3) g_corp_acc+=(uint64_t)k;if(m->has_mtp && k>=1) mtp_absorb(m, all+kv+1, m->h_all, k, kv); /* KV MTP in sync coi verificati *//* hlast deve corrispondere all'ultima posizione ACCETTATA (kv+k), non a fine batch */if(m->h_all && k<S-1) memcpy(m->hlast, m->h_all+(int64_t)k*m->c.hidden, m->c.hidden*sizeof(float));@@ -5215,6 +5289,9 @@ static void run_text(Model *m, const char *snap, const char *prompt, int ngen){printf("speculation: %.2f tokens/forward (%llu forwards per %llu tokens) | MTP acceptance %.0f%% (%llu/%llu)\n",m->n_fw?(double)m->n_emit/m->n_fw:1.0, (unsigned long long)m->n_fw, (unsigned long long)m->n_emit,m->mtp_prop?100.0*m->mtp_acc/m->mtp_prop:0.0, (unsigned long long)m->mtp_acc, (unsigned long long)m->mtp_prop);+ if(g_corp_prop) printf("corpus drafts: %.0f%% acceptance (%llu/%llu proposed from %ld frozen ids)\n",+ 100.0*g_corp_acc/g_corp_prop, (unsigned long long)g_corp_acc,+ (unsigned long long)g_corp_prop, g_corp_n);if(g_cp_enq) printf("couple: %ld cross-layer prefetch hints enqueued\n", g_cp_enq);if(g_grd.prop) printf("grammar: %.0f%% acceptance (%llu/%llu forced drafts)\n",100.0*g_grd.acc/g_grd.prop, (unsigned long long)g_grd.acc, (unsigned long long)g_grd.prop);@@ -6893,6 +6970,7 @@ int main(int argc, char **argv){if(g_mirror_dir&&!*g_mirror_dir) g_mirror_dir = NULL;g_idot = getenv("IDOT")?atoi(getenv("IDOT")):1; /* 0 = kernel f32 esatti (A/B) */g_spec_pin = getenv("SPEC_PIN")?atoi(getenv("SPEC_PIN")):1; /* #163: 0 = gate S-dipendenti storici / legacy S-dependent gates */+ corpus_load(); /* COLI_DRAFT_CORPUS: external draft source */if(getenv("ROUTE_TRACE")&&*getenv("ROUTE_TRACE")){g_route_fp=fopen(getenv("ROUTE_TRACE"),"w");if(!g_route_fp) fprintf(stderr,"[ROUTE_TRACE] cannot open %s\n",getenv("ROUTE_TRACE"));diff --git a/c/tests/test_corpus_draft.c b/c/tests/test_corpus_draft.cnew file mode 100644index 00000000..9508c457--- /dev/null+++ b/c/tests/test_corpus_draft.c@@ -0,0 +1,97 @@+/* Corpus draft source (COLI_DRAFT_CORPUS): the proposal lookup must be a pure+ * suffix match over the frozen ids, and it must never propose anything the+ * verifier would then have to reject for a structural reason.+ *+ * Why this test exists: corpus_draft() is the only NEW way tokens can enter+ * spec_decode's draft buffer. The verification path downstream is unchanged and+ * already lossless, but that guarantee only holds if the proposals themselves+ * are well-formed: within bounds, never past a span separator (-1), never+ * longer than the caller's cap, and empty when no suffix matches. A malformed+ * proposal would index draft[]/batch[] out of range in spec_decode long before+ * verification ever ran.+ *+ * The contract under test:+ * 1. longest-suffix wins: an n=8 match is preferred over an n=3 one+ * 2. most-recent-first: with two matches of equal length, the later corpus+ * occurrence is proposed (recency = the freshest continuation)+ * 3. the -1 span separator terminates a proposal (spans never bleed together)+ * 4. g <= k always, and g == 0 when the context suffix is absent+ * 5. short contexts (< minn) and empty corpora propose nothing+ *+ * In-memory only (no scratch files, no model), so it builds clean on the+ * Windows MinGW CI job. */+#define main coli_glm_main_unused+#include "../colibri.c"+#undef main++static int g_nfails = 0;++static void check(int cond, const char *what){+ if(!cond){ printf("FAIL: %s\n", what); g_nfails++; }+}++/* corpus_draft reads the file-scope g_corp/g_corp_n; set them directly so the+ * test needs no filesystem (corpus_load's parsing is exercised end-to-end by+ * the engine itself, this is about the lookup). */+static void set_corpus(int *ids, long n){ g_corp = ids; g_corp_n = n; }++int main(void){+ int out[64];++ /* 1. exact suffix match proposes the continuation, longest suffix wins.+ * corpus: ... 10 11 12 13 [14 15 16] ... and a decoy short match of "13". */+ { static int c[] = {90,91,13,77,78, 10,11,12,13,14,15,16,17};+ set_corpus(c, (long)(sizeof(c)/sizeof(c[0])));+ int ctx[] = {1,2,3,10,11,12,13}; /* 4-long suffix 10 11 12 13 */+ int g = corpus_draft(ctx, 7, out, 4, 3, 8);+ check(g == 4, "longest-suffix match proposes k tokens");+ check(g >= 1 && out[0] == 14, "proposal continues the matched span (14)");+ if(g == 4) check(out[1]==15 && out[2]==16 && out[3]==17, "proposal is contiguous");+ /* the decoy (13 at index 2) must NOT win: its continuation is 77 */+ check(!(g >= 1 && out[0] == 77), "longer suffix beats the shorter decoy");+ }++ /* 2. recency: two identical 3-long matches, the later one is proposed. */+ { static int c[] = {7,8,9,100,101, 7,8,9,200,201};+ set_corpus(c, (long)(sizeof(c)/sizeof(c[0])));+ int ctx[] = {0,7,8,9};+ int g = corpus_draft(ctx, 4, out, 2, 3, 8);+ check(g == 2, "recency case proposes k tokens");+ check(g >= 1 && out[0] == 200, "most recent occurrence wins (200, not 100)");+ }++ /* 3. the span separator (-1) stops a proposal: no bleed across spans. */+ { static int c[] = {5,6,7,42,-1, 99,98};+ set_corpus(c, (long)(sizeof(c)/sizeof(c[0])));+ int ctx[] = {4,5,6,7};+ int g = corpus_draft(ctx, 4, out, 8, 3, 8);+ check(g == 1, "proposal stops at the -1 span separator");+ check(g >= 1 && out[0] == 42, "the pre-separator token is still proposed");+ }++ /* 4. cap and absence. */+ { static int c[] = {1,2,3,4,5,6,7,8,9,10,11,12};+ set_corpus(c, (long)(sizeof(c)/sizeof(c[0])));+ int ctx[] = {1,2,3};+ int g = corpus_draft(ctx, 3, out, 3, 3, 8);+ check(g <= 3, "proposal never exceeds the caller's cap");+ int ctx2[] = {555,556,557};+ check(corpus_draft(ctx2, 3, out, 8, 3, 8) == 0, "absent suffix proposes nothing");+ }++ /* 5. degenerate inputs are silent, not crashes. */+ { static int c[] = {1,2,3,4,5,6,7,8,9,10};+ set_corpus(c, (long)(sizeof(c)/sizeof(c[0])));+ int ctx[] = {1,2};+ check(corpus_draft(ctx, 2, out, 8, 3, 8) == 0, "context shorter than minn proposes nothing");+ check(corpus_draft(ctx, 2, out, 0, 3, 8) == 0, "k=0 proposes nothing");+ set_corpus(NULL, 0);+ int ctx3[] = {1,2,3,4};+ check(corpus_draft(ctx3, 4, out, 8, 3, 8) == 0, "empty corpus proposes nothing");+ }++ set_corpus(NULL, 0); /* don't free static storage */+ if(g_nfails){ printf("corpus_draft: %d FAILED\n", g_nfails); return 1; }+ printf("corpus_draft: suffix lookup, recency, span separator, caps ok\n");+ return 0;+}diff --git a/docs/ENVIRONMENT.md b/docs/ENVIRONMENT.mdindex b2328591..e9f8069e 100644--- a/docs/ENVIRONMENT.md+++ b/docs/ENVIRONMENT.md@@ -137,6 +137,9 @@ These are for testing, benchmarking, or internal use β not part of the everyda| `GRAMMAR` | unset | Path to a GBNF grammar file to constrain generation. Takes precedence over `SCHEMA`. || `SCHEMA` | unset | Path to a JSON-Schema file compiled to GBNF to constrain generation (consulted only when `GRAMMAR` is empty). || `GRAMMAR_DRAFT` | unset | Max grammar-forced draft span length. |+| `COLI_DRAFT_CORPUS` | unset | Path to a file of frozen token ids (whitespace-separated, `-1` separates spans) used as a speculative draft source: the engine proposes the continuation that followed the longest suffix of the live context found in the corpus. Off when unset. Build one from any run with `TOKENS=1`. See [corpus-draft.md](corpus-draft.md). |+| `COLI_CORPUS_K` | `8` (max 48) | Proposal depth for `COLI_DRAFT_CORPUS`. Deeper raises the forward multiplier and the per-forward cost. |+| `COLI_CORPUS_MINACC` | `50` | Acceptance floor (percent) for the corpus source. Below it over a 24-proposal window the source pauses for 256 tokens, then re-arms β rejected drafts cost real time. || `EXPERT_BUDGET` | `0` (off) | Cap experts loaded per layer (MoE-Spec). **Quarantined:** silently forced to `0` unless `EXPERT_BUDGET_EXPERIMENTAL` is set β every tested value is either no faster or incoherent (issue #303). || `EXPERT_BUDGET_EXPERIMENTAL` | unset | Setting it (any value) allows `EXPERT_BUDGET>0` to actually take effect (expect garbage, #294). || `DSA` | on | Dynamic Sparse Attention indexer. `DSA=0` disables. |diff --git a/docs/corpus-draft.md b/docs/corpus-draft.mdnew file mode 100644index 00000000..b4724edc--- /dev/null+++ b/docs/corpus-draft.md@@ -0,0 +1,87 @@+# Frozen-corpus speculative drafts++*The canonical reference for the `COLI_DRAFT_CORPUS=` draft source. It is the+existing n-gram draft (method E) taken off the live context and pointed at a+persistent file of previously generated token ids β the retrieval-based+speculative decoding idea, reusing this engine's verification unchanged.*++## The mechanism in one paragraph++`ngram_draft()` already proposes a continuation by matching the last two tokens+of the **current** sequence against earlier positions in that same sequence. A+frozen corpus generalizes that: the engine loads a file of token ids produced by+past runs, and at each decode step proposes the continuation that followed the+**longest suffix of the live context** found anywhere in that corpus (suffix+lengths 8 down to 3, most recent occurrence first). The proposal enters the same+batch-union verify forward as MTP, n-gram and grammar drafts, so it is only ever+a *proposal*: the target model's own argmax decides what is accepted, and the+output is what the engine would have produced without any drafting. Unlike MTP+(one token per forward) a corpus hit proposes a whole span at once.++## What it costs and what it pays++Measured on GLM-5.2 int4, 96-token greedy decode, replaying a prompt whose+generation is in the corpus:++| Host | Baseline (MTP) | With corpus drafts |+|---|---|---|+| H200, fully resident | 1.78 tok/forward, 3.69 tok/s | 6.00 tok/forward, **4.52 tok/s** (+22%), 90% acceptance |+| CPU (24C Xeon, 120 GB pin) | 1.76 tok/forward, 0.82 tok/s | 7.92 tok/forward, **1.00 tok/s** (+22%), 100% acceptance |++Note the gap between the two multipliers: **3β4Γ fewer forwards buys ~22% of+wall clock, not 3β4Γ**. In an MoE every row of a verify batch activates its own+experts, so speculation amortizes the dense path, attention and per-forward+overhead, but never the expert work itself β that scales with rows. The same+ceiling showed up on a disk-streaming host, where the extra rows also widen the+per-forward expert union and cost LRU hit rate. Treat the forward count as the+thing being optimized and measure wall clock separately.++Drafts that are *rejected* cost real time, so the source pauses itself below+`COLI_CORPUS_MINACC` acceptance (default 50%, the measured break-even: 90%+acceptance gave +22%, 19% gave β25%). A prompt with no relation to the corpus+costs nothing at all β the suffix match simply finds nothing and the source+stays inert (measured: identical forward count to a run with no corpus). The+risk lives in the *neighbourhood* of the corpus, where near-miss prefixes+produce spurious matches; that is what the guard is for.++## Usage++```bash+# 1. freeze a run's output ids (TOKENS=1 already dumps them to stderr)+TOKENS=1 COLI_TEMP=0 ./coli run --ngen 96 "..." 2>&1 |+ sed -n 's/.*\[TOKENS\][0-9 ]*generated://p' > corpus.txt++# 2. draft from it on later runs+COLI_DRAFT_CORPUS=corpus.txt COLI_CORPUS_K=8 COLI_TEMP=0 ./coli run --ngen 96 "..."+```++- `COLI_DRAFT_CORPUS=<file>` β whitespace-separated token ids. `-1` separates+ spans; a proposal never crosses one. Absent or unreadable = source off.+- `COLI_CORPUS_K=n` β proposal depth (default 8, capped at 48 by the verify+ batch). Deeper proposals raise the forward multiplier and the per-forward+ cost; 8 was the best measured trade on GLM-5.2.+- `COLI_CORPUS_MINACC=pct` β pause threshold (default 50). Below this acceptance+ over a 24-proposal window the source pauses for 256 tokens, then re-arms.+- The engine reports at end of run:+ `corpus drafts: NN% acceptance (a/b proposed from N frozen ids)`.++The source is **off by default** and takes priority over MTP when it has a+proposal; MTP fills the gaps where the corpus has no match.++## Losslessness++The corpus never constrains sampling β it only proposes, and the existing verify+loop accepts a token solely when it matches what the model itself produced. Two+independent checks:++- the token-exact oracle passes with the source **armed** (`SNAP=./glm_tiny+ COLI_DRAFT_CORPUS=... ./colibri 64 16 16` β 20/20, same as unarmed);+- a 96-token CPU generation with 100% acceptance at depth 8 is **byte-identical**+ to the same generation with the source off.++One caveat, and it is not specific to this source: on the CUDA path a run whose+drafts are accepted in long streaks can diverge from the unbatched path by a+single near-tie token (measured: 1 token in 96, at position 85, first 84+identical). Deep verify batches are not bit-identical to S=1 forwards there, and+the same effect shows up as a lower acceptance rate on GPU (90%) than on CPU+(100%) for the identical corpus. The CPU path is byte-exact.