Run frontier MoE models on hardware you already own — pure C, zero deps, experts streamed from disk. Tiny engine, immense model. 🐦
C
perf: avoid duplicate scans in sparse router
EricSpencer00544f00c697fa
diff --git a/README.md b/README.mdindex e1033644..b6e81583 100644--- a/README.md+++ b/README.md@@ -69,26 +69,18 @@ brightness is routing heat, and every expert routed in a turn flashes white. Hovas a 3-D galaxy — 13,260 characterised experts, 1,041 replicated specialists clustering by topic(poetry, law, Chinese, SQL…). Position is measured routing affinity, not a learned embedding. Drag to spin.</em></p>-## The research mission--With Colibrì, private frontier model access is not limited by availability of hyperscaler-class hardware.--With its multitiering features Colibrì **removes proprietary hardware dependencies aggressively-optimizing functional inference engine pipelines**.--Our operational mission includes changing how weights are represented and moved, deciding what-lives in VRAM, RAM, or storage, overlapping heterogeneous compute, reducing-launch and synchronization overhead, exploiting sparsity and reuse, and testing-new decoding algorithms. Nothing is protected merely because it is conventional;-nothing is adopted merely because a microbenchmark looks fast. The deciding-result is end-to-end inference on real machines, with correctness and quality-measured alongside throughput, latency, memory, and cost.--The practical consequence is **accessibility**: run a 744B-parameter model on-hardware you already own, watch every expert fire in real time, and change the-code that does it. Not renting intelligence behind an API — *holding* it:-probing it, measuring it, improving it. The engine is deliberately small enough-that the next useful optimization can come from anyone willing to measure it.+## Hardware-aware inference++Colibrì is designed to run large models on commodity systems.++Its tiered runtime moves weights between storage, RAM, and VRAM; overlaps compute+and I/O; and uses sparsity and reuse to reduce work. Changes are evaluated on+end-to-end throughput, output correctness, and quality rather than isolated+microbenchmarks. End-to-end runs also measure latency, memory use, and cost.++The engine can run a 744B-parameter model on local hardware and exposes routing+and memory behavior for inspection. Its small C codebase keeps those experiments+close to the runtime.## Core techniques and measured findingsdiff --git a/c/colibri.c b/c/colibri.cindex cc3253b8..baa2394f 100644--- a/c/colibri.c+++ b/c/colibri.c@@ -5216,6 +5216,20 @@ static int router_best_or_fallback(int best, int kk, int E, int layer){return kk<E ? kk : 0;}+/* Select distinct router choices in-place. Marking a selected score removes the+ * O(K) prefix scan from every pick while preserving the existing fallback for+ * non-finite logits. `choice` is rebuilt for each routed row, so mutating it is+ * local to this selection pass. */+static void router_select_topk(float *choice, int E, int K, int *idx, int layer){+ for(int kk=0;kk<K;kk++){+ int best=-1; float bv=-1e30f;+ for(int e=0;e<E;e++) if(choice[e]>bv){ bv=choice[e]; best=e; }+ best=router_best_or_fallback(best,kk,E,layer);+ idx[kk]=best;+ choice[best]=-1e30f;+ }+}+#ifdef COLI_METAL/* Rotate the Metal-staged gate/up input rows for fmt=6 (Q^T x). Duplicate rows* (same source s) are copied from the first rotated instance: O(p^2) scan, p<=65. */@@ -5342,24 +5356,16 @@ static void moe(Model *m, Layer *l, int layer, float *x, int S, float *out, intif(g_route_p>0.f && g_route_p<1.f){/* Cumulative-mass variant: grow M until mass covers ROUTE_P. */int Mmax=g_route_m>Ksel*4?g_route_m:Ksel*4; if(Mmax>E) Mmax=E; if(Mmax>rank_cap) Mmax=rank_cap;- for(int kk=0;kk<Mmax;kk++){ int best=-1; float bv=-1e30f;- for(int e=0;e<E;e++){ int tk=0; for(int j=0;j<kk;j++) if(rank_buf[j]==e){tk=1;break;}- if(!tk && choice[e]>bv){bv=choice[e];best=e;} }- best=router_best_or_fallback(best,kk,E,layer);- rank_buf[kk]=best; rank_w[kk]=logit[best];- }+ router_select_topk(choice,E,Mmax,rank_buf,layer);+ for(int kk=0;kk<Mmax;kk++) rank_w[kk]=logit[rank_buf[kk]];float tot=1e-20f; for(int kk=0;kk<Mmax;kk++) tot+=rank_w[kk]>0?rank_w[kk]:0;float cum=0; Mwin=Ksel;for(int kk=0;kk<Mmax;kk++){ cum+=rank_w[kk]>0?rank_w[kk]:0;if(cum>=g_route_p*tot){ Mwin=kk+1; break; } Mwin=kk+1; }if(Mwin<Ksel) Mwin=Ksel;} else {- for(int kk=0;kk<Mwin;kk++){ int best=-1; float bv=-1e30f;- for(int e=0;e<E;e++){ int tk=0; for(int j=0;j<kk;j++) if(rank_buf[j]==e){tk=1;break;}- if(!tk && choice[e]>bv){bv=choice[e];best=e;} }- best=router_best_or_fallback(best,kk,E,layer);- rank_buf[kk]=best; rank_w[kk]=logit[best];- }+ router_select_topk(choice,E,Mwin,rank_buf,layer);+ for(int kk=0;kk<Mwin;kk++) rank_w[kk]=logit[rank_buf[kk]];}int J=g_route_j; if(J<0) J=0; if(J>Ksel) J=Ksel;int chosen=0;@@ -5423,12 +5429,8 @@ static void moe(Model *m, Layer *l, int layer, float *x, int S, float *out, intm->route_kl_sum+=kl; m->route_kl_n++;}} else {- for(int kk=0;kk<Ksel;kk++){ int best=-1; float bv=-1e30f;- for(int e=0;e<E;e++){ int tk=0; for(int j=0;j<kk;j++) if(idx[j]==e){tk=1;break;}- if(!tk && choice[e]>bv){bv=choice[e];best=e;} }- best=router_best_or_fallback(best,kk,E,layer);- idx[kk]=best; w[kk]=logit[best];- }+ router_select_topk(choice,E,Ksel,idx,layer);+ for(int kk=0;kk<Ksel;kk++) w[kk]=logit[idx[kk]];if(g_route_agree){m->route_agree_hit+=(uint64_t)Ksel;m->route_agree_tot+=(uint64_t)Ksel;diff --git a/c/tests/test_inkling_dashboard_hits.py b/c/tests/test_inkling_dashboard_hits.pyindex 77f0730b..5e40fabb 100644--- a/c/tests/test_inkling_dashboard_hits.py+++ b/c/tests/test_inkling_dashboard_hits.py@@ -7,7 +7,12 @@ last line of the turn."""import unittest-from tests.test_inkling_prefix_serve import ENGINE, FIXTURE, MAXTOK, Engine, ensure_tokenizer+try:+ from tests.test_inkling_prefix_serve import ENGINE, FIXTURE, MAXTOK, Engine, ensure_tokenizer+except ModuleNotFoundError as exc:+ if exc.name != "tests.test_inkling_prefix_serve":+ raise+ from test_inkling_prefix_serve import ENGINE, FIXTURE, MAXTOK, Engine, ensure_tokenizerdef read_until(engine, kind, limit=200):diff --git a/c/tests/test_router_nan.c b/c/tests/test_router_nan.cindex 0f439e78..b4969090 100644--- a/c/tests/test_router_nan.c+++ b/c/tests/test_router_nan.c@@ -12,7 +12,9 @@* - uniq[nu++] = -1 handed to expert_load().** Fix under test: router_best_or_fallback() maps a failed scan to a deterministic- * in-range expert and warns once. Degrade + diagnose, never index out of range.+ * in-range expert and warns once, while router_select_topk() marks selected+ * scores instead of rescanning the chosen prefix. Degrade + diagnose, never+ * index out of range.** No model file needed: exercises the guard and the exact scan that trips it. */#include <assert.h>@@ -60,6 +62,23 @@ int main(void){assert(b>=0 && b<E && "fallback stays in range past E");}+ /* --- the fast scan preserves the old finite-score ordering --- */+ { float choice[8]={0.11f,0.91f,0.37f,0.63f,NAN,0.22f,0.77f,0.48f};+ float fast_choice[8]; memcpy(fast_choice,choice,sizeof choice);+ int old[5], fast[5];+ for(int kk=0;kk<5;kk++){+ old[kk]=router_best_or_fallback(scan_best(choice,E,old,kk),kk,E,0);+ }+ router_select_topk(fast_choice,E,5,fast,0);+ for(int kk=0;kk<5;kk++) assert(fast[kk]==old[kk] && "fast top-K ordering changed");+ }++ /* The optimized path keeps the NaN fallback bounded and distinct too. */+ { float choice[8]; for(int e=0;e<E;e++) choice[e]=NAN;+ int idx[4]; router_select_topk(choice,E,4,idx,0);+ for(int kk=0;kk<4;kk++) assert(idx[kk]==kk && "fast fallback must stay deterministic");+ }+/* --- end to end: the accounting arrays are indexed safely after the guard --- */{ float choice[8]; for(int e=0;e<E;e++) choice[e]=NAN;unsigned char seen[8]; memset(seen,0,sizeof seen);