Run frontier MoE models on hardware you already own — pure C, zero deps, experts streamed from disk. Tiny engine, immense model. 🐦
C
fix(cuda): budget expert scale allocations separately
ZacharyZcR1d6bfe2e3d73
diff --git a/c/Makefile b/c/Makefileindex 341ad971..a36104de 100644--- a/c/Makefile+++ b/c/Makefile@@ -2180,3 +2180,7 @@ tests/test_kv_prefix$(EXE): tests/test_kv_prefix.c kv_prefix.htests/test_kimi_request_state$(EXE): tests/test_kimi_request_state.c kimi_k3.c kv_prefix.h serve_codec.h st.h json.h tok.h tok_unicode.h tok_unicode_o200k.h compat.h quant.h idot.h omp_tune.h route_trace.h$(CC) $(NOCUDA_CFLAGS) tests/test_kimi_request_state.c $(VK_OBJ) -o tests/test_kimi_request_state$(EXE) $(NOCUDA_LDFLAGS)++# Budget separate gate/up/down scale allocations at their own size classes.+tests/test_qwen36_tier_scale_budget$(EXE): tests/test_qwen36_tier_scale_budget.c tests/qwen36_fake_cuda.h qwen36_tier.c qwen36_tier.h backend_cuda.h compat.h+ $(CC) $(CFLAGS) -DCOLI_CUDA $< -o $@ $(LDFLAGS)diff --git a/c/qwen36_tier.c b/c/qwen36_tier.cindex 2b2e24e5..179489b2 100644--- a/c/qwen36_tier.c+++ b/c/qwen36_tier.c@@ -609,8 +609,9 @@ int qt_init(int nl, int ne, int D, int Ih, int cap, int topk, int expert_gs,* is the resident count. The 22-28 % the granularity costs is real; only* pooling experts into one arena per device would win it back (open). */size_t mat_bytes = G.wfmt==4 ? (size_t)D*Ih/2 : (size_t)D*Ih;- size_t scl_bytes = (2*G.sc_gu+G.sc_d)/3*sizeof(float);- G.exp_bytes = 3*dev_alloc_footprint(mat_bytes) + 3*dev_alloc_footprint(scl_bytes); /* + allocation slack */+ G.exp_bytes = 3*dev_alloc_footprint(mat_bytes)+ + 2*dev_alloc_footprint(G.sc_gu*sizeof(float))+ + dev_alloc_footprint(G.sc_d*sizeof(float));/* Per-device allowance for tier + trunk: CUDA_EXPERT_GB when numeric,* else free minus 1 GB headroom. The heat table is loaded here too (itdiff --git a/c/tests/test_qwen36_tier_scale_budget.c b/c/tests/test_qwen36_tier_scale_budget.cnew file mode 100644index 00000000..bb2ef81e--- /dev/null+++ b/c/tests/test_qwen36_tier_scale_budget.c@@ -0,0 +1,34 @@+/* Separate scale allocations can cross different allocator size classes. */+#include <stdio.h>+#include "../compat.h"+#include "qwen36_fake_cuda.h"+#include "../qwen36_tier.c"+int main(void) {+ /* D, Ih, group size (0 = int8), bytes/expert, allowance, admitted experts.+ * Constants include three weight and three separate scale allocations. */+ const struct { int D,Ih,gs; size_t bytes,budget; int planned; } cases[]={+ {64,2048,0,458752,917503,1},+ {2048,64,0,450560,901119,1},+ {64,3000,0,655360,1310720,2},+ {64,64,0,49152,98304,2},+ {96,2048,64,385024,770047,1},+ };+ setenv("COLI_CUDA","1",1); setenv("COLI_GPUS","0",1);+ setenv("QT_NO_WARMSTART","1",1); setenv("HEAT_FILE","",1);+ for(size_t i=0;i<sizeof cases/sizeof *cases;i++) {+ char gb[64];snprintf(gb,sizeof gb,"%.17g",cases[i].budget/1073741824.0);+ setenv("CUDA_EXPERT_GB",gb,1);+ if(!qt_init(1,3,cases[i].D,cases[i].Ih,3,1,cases[i].gs,cases[i].gs!=0)) return 1;+ int layers[3],eids[3],n=qt_plan_fill(layers,eids,3);+ int ok=G.exp_bytes==cases[i].bytes && n==cases[i].planned &&+ G.used[0]==cases[i].bytes*n && G.used[0]<=G.budget[0];+ if(!ok) fprintf(stderr,"case %zu: bytes=%zu expected=%zu planned=%d expected=%d\n",+ i,G.exp_bytes,cases[i].bytes,n,cases[i].planned);+ qt_shutdown();+ /* Accommodate dev's retained host storage until teardown cleanup lands. */+ free(G.slot);G.slot=NULL;free(G.is_x);G.is_x=NULL;+ free(G.fill_order);G.fill_order=NULL;free(G.heat0);G.heat0=NULL;+ if(!ok) return 1;+ }+ puts("tier scale budget: ok");return 0;+}