Skip to content

Run frontier MoE models on hardware you already own — pure C, zero deps, experts streamed from disk. Tiny engine, immense model. 🐦

C

perf: avoid duplicate scans in sparse router

EricSpencer00544f00c697fa
diff --git a/README.md b/README.md
index e1033644..b6e81583 100644
--- a/README.md
+++ b/README.md
@@ -69,26 +69,18 @@ brightness is routing heat, and every expert routed in a turn flashes white. Hov
as a 3-D galaxy — 13,260 characterised experts, 1,041 replicated specialists clustering by topic
(poetry, law, Chinese, SQL…). Position is measured routing affinity, not a learned embedding. Drag to spin.</em></p>
-## The research mission
-
-With Colibrì, private frontier model access is not limited by availability of hyperscaler-class hardware.
-
-With its multitiering features Colibrì **removes proprietary hardware dependencies aggressively
-optimizing functional inference engine pipelines**.
-
-Our operational mission includes changing how weights are represented and moved, deciding what
-lives in VRAM, RAM, or storage, overlapping heterogeneous compute, reducing
-launch and synchronization overhead, exploiting sparsity and reuse, and testing
-new decoding algorithms. Nothing is protected merely because it is conventional;
-nothing is adopted merely because a microbenchmark looks fast. The deciding
-result is end-to-end inference on real machines, with correctness and quality
-measured alongside throughput, latency, memory, and cost.
-
-The practical consequence is **accessibility**: run a 744B-parameter model on
-hardware you already own, watch every expert fire in real time, and change the
-code that does it. Not renting intelligence behind an API — *holding* it:
-probing it, measuring it, improving it. The engine is deliberately small enough
-that the next useful optimization can come from anyone willing to measure it.
+## Hardware-aware inference
+
+Colibrì is designed to run large models on commodity systems.
+
+Its tiered runtime moves weights between storage, RAM, and VRAM; overlaps compute
+and I/O; and uses sparsity and reuse to reduce work. Changes are evaluated on
+end-to-end throughput, output correctness, and quality rather than isolated
+microbenchmarks. End-to-end runs also measure latency, memory use, and cost.
+
+The engine can run a 744B-parameter model on local hardware and exposes routing
+and memory behavior for inspection. Its small C codebase keeps those experiments
+close to the runtime.
## Core techniques and measured findings
diff --git a/c/colibri.c b/c/colibri.c
index cc3253b8..baa2394f 100644
--- a/c/colibri.c
+++ b/c/colibri.c
@@ -5216,6 +5216,20 @@ static int router_best_or_fallback(int best, int kk, int E, int layer){
return kk<E ? kk : 0;
}
+/* Select distinct router choices in-place. Marking a selected score removes the
+ * O(K) prefix scan from every pick while preserving the existing fallback for
+ * non-finite logits. `choice` is rebuilt for each routed row, so mutating it is
+ * local to this selection pass. */
+static void router_select_topk(float *choice, int E, int K, int *idx, int layer){
+ for(int kk=0;kk<K;kk++){
+ int best=-1; float bv=-1e30f;
+ for(int e=0;e<E;e++) if(choice[e]>bv){ bv=choice[e]; best=e; }
+ best=router_best_or_fallback(best,kk,E,layer);
+ idx[kk]=best;
+ choice[best]=-1e30f;
+ }
+}
+
#ifdef COLI_METAL
/* Rotate the Metal-staged gate/up input rows for fmt=6 (Q^T x). Duplicate rows
* (same source s) are copied from the first rotated instance: O(p^2) scan, p<=65. */
@@ -5342,24 +5356,16 @@ static void moe(Model *m, Layer *l, int layer, float *x, int S, float *out, int
if(g_route_p>0.f && g_route_p<1.f){
/* Cumulative-mass variant: grow M until mass covers ROUTE_P. */
int Mmax=g_route_m>Ksel*4?g_route_m:Ksel*4; if(Mmax>E) Mmax=E; if(Mmax>rank_cap) Mmax=rank_cap;
- for(int kk=0;kk<Mmax;kk++){ int best=-1; float bv=-1e30f;
- for(int e=0;e<E;e++){ int tk=0; for(int j=0;j<kk;j++) if(rank_buf[j]==e){tk=1;break;}
- if(!tk && choice[e]>bv){bv=choice[e];best=e;} }
- best=router_best_or_fallback(best,kk,E,layer);
- rank_buf[kk]=best; rank_w[kk]=logit[best];
- }
+ router_select_topk(choice,E,Mmax,rank_buf,layer);
+ for(int kk=0;kk<Mmax;kk++) rank_w[kk]=logit[rank_buf[kk]];
float tot=1e-20f; for(int kk=0;kk<Mmax;kk++) tot+=rank_w[kk]>0?rank_w[kk]:0;
float cum=0; Mwin=Ksel;
for(int kk=0;kk<Mmax;kk++){ cum+=rank_w[kk]>0?rank_w[kk]:0;
if(cum>=g_route_p*tot){ Mwin=kk+1; break; } Mwin=kk+1; }
if(Mwin<Ksel) Mwin=Ksel;
} else {
- for(int kk=0;kk<Mwin;kk++){ int best=-1; float bv=-1e30f;
- for(int e=0;e<E;e++){ int tk=0; for(int j=0;j<kk;j++) if(rank_buf[j]==e){tk=1;break;}
- if(!tk && choice[e]>bv){bv=choice[e];best=e;} }
- best=router_best_or_fallback(best,kk,E,layer);
- rank_buf[kk]=best; rank_w[kk]=logit[best];
- }
+ router_select_topk(choice,E,Mwin,rank_buf,layer);
+ for(int kk=0;kk<Mwin;kk++) rank_w[kk]=logit[rank_buf[kk]];
}
int J=g_route_j; if(J<0) J=0; if(J>Ksel) J=Ksel;
int chosen=0;
@@ -5423,12 +5429,8 @@ static void moe(Model *m, Layer *l, int layer, float *x, int S, float *out, int
m->route_kl_sum+=kl; m->route_kl_n++;
}
} else {
- for(int kk=0;kk<Ksel;kk++){ int best=-1; float bv=-1e30f;
- for(int e=0;e<E;e++){ int tk=0; for(int j=0;j<kk;j++) if(idx[j]==e){tk=1;break;}
- if(!tk && choice[e]>bv){bv=choice[e];best=e;} }
- best=router_best_or_fallback(best,kk,E,layer);
- idx[kk]=best; w[kk]=logit[best];
- }
+ router_select_topk(choice,E,Ksel,idx,layer);
+ for(int kk=0;kk<Ksel;kk++) w[kk]=logit[idx[kk]];
if(g_route_agree){
m->route_agree_hit+=(uint64_t)Ksel;
m->route_agree_tot+=(uint64_t)Ksel;
diff --git a/c/tests/test_inkling_dashboard_hits.py b/c/tests/test_inkling_dashboard_hits.py
index 77f0730b..5e40fabb 100644
--- a/c/tests/test_inkling_dashboard_hits.py
+++ b/c/tests/test_inkling_dashboard_hits.py
@@ -7,7 +7,12 @@ last line of the turn.
"""
import unittest
-from tests.test_inkling_prefix_serve import ENGINE, FIXTURE, MAXTOK, Engine, ensure_tokenizer
+try:
+ from tests.test_inkling_prefix_serve import ENGINE, FIXTURE, MAXTOK, Engine, ensure_tokenizer
+except ModuleNotFoundError as exc:
+ if exc.name != "tests.test_inkling_prefix_serve":
+ raise
+ from test_inkling_prefix_serve import ENGINE, FIXTURE, MAXTOK, Engine, ensure_tokenizer
def read_until(engine, kind, limit=200):
diff --git a/c/tests/test_router_nan.c b/c/tests/test_router_nan.c
index 0f439e78..b4969090 100644
--- a/c/tests/test_router_nan.c
+++ b/c/tests/test_router_nan.c
@@ -12,7 +12,9 @@
* - uniq[nu++] = -1 handed to expert_load().
*
* Fix under test: router_best_or_fallback() maps a failed scan to a deterministic
- * in-range expert and warns once. Degrade + diagnose, never index out of range.
+ * in-range expert and warns once, while router_select_topk() marks selected
+ * scores instead of rescanning the chosen prefix. Degrade + diagnose, never
+ * index out of range.
*
* No model file needed: exercises the guard and the exact scan that trips it. */
#include <assert.h>
@@ -60,6 +62,23 @@ int main(void){
assert(b>=0 && b<E && "fallback stays in range past E");
}
+ /* --- the fast scan preserves the old finite-score ordering --- */
+ { float choice[8]={0.11f,0.91f,0.37f,0.63f,NAN,0.22f,0.77f,0.48f};
+ float fast_choice[8]; memcpy(fast_choice,choice,sizeof choice);
+ int old[5], fast[5];
+ for(int kk=0;kk<5;kk++){
+ old[kk]=router_best_or_fallback(scan_best(choice,E,old,kk),kk,E,0);
+ }
+ router_select_topk(fast_choice,E,5,fast,0);
+ for(int kk=0;kk<5;kk++) assert(fast[kk]==old[kk] && "fast top-K ordering changed");
+ }
+
+ /* The optimized path keeps the NaN fallback bounded and distinct too. */
+ { float choice[8]; for(int e=0;e<E;e++) choice[e]=NAN;
+ int idx[4]; router_select_topk(choice,E,4,idx,0);
+ for(int kk=0;kk<4;kk++) assert(idx[kk]==kk && "fast fallback must stay deterministic");
+ }
+
/* --- end to end: the accounting arrays are indexed safely after the guard --- */
{ float choice[8]; for(int e=0;e<E;e++) choice[e]=NAN;
unsigned char seen[8]; memset(seen,0,sizeof seen);