Skip to content

Run frontier MoE models on hardware you already own — pure C, zero deps, experts streamed from disk. Tiny engine, immense model. 🐦

C

fix(cuda): budget expert scale allocations separately

ZacharyZcR1d6bfe2e3d73
diff --git a/c/Makefile b/c/Makefile
index 341ad971..a36104de 100644
--- a/c/Makefile
+++ b/c/Makefile
@@ -2180,3 +2180,7 @@ tests/test_kv_prefix$(EXE): tests/test_kv_prefix.c kv_prefix.h
tests/test_kimi_request_state$(EXE): tests/test_kimi_request_state.c kimi_k3.c kv_prefix.h serve_codec.h st.h json.h tok.h tok_unicode.h tok_unicode_o200k.h compat.h quant.h idot.h omp_tune.h route_trace.h
$(CC) $(NOCUDA_CFLAGS) tests/test_kimi_request_state.c $(VK_OBJ) -o tests/test_kimi_request_state$(EXE) $(NOCUDA_LDFLAGS)
+
+# Budget separate gate/up/down scale allocations at their own size classes.
+tests/test_qwen36_tier_scale_budget$(EXE): tests/test_qwen36_tier_scale_budget.c tests/qwen36_fake_cuda.h qwen36_tier.c qwen36_tier.h backend_cuda.h compat.h
+ $(CC) $(CFLAGS) -DCOLI_CUDA $< -o $@ $(LDFLAGS)
diff --git a/c/qwen36_tier.c b/c/qwen36_tier.c
index 2b2e24e5..179489b2 100644
--- a/c/qwen36_tier.c
+++ b/c/qwen36_tier.c
@@ -609,8 +609,9 @@ int qt_init(int nl, int ne, int D, int Ih, int cap, int topk, int expert_gs,
* is the resident count. The 22-28 % the granularity costs is real; only
* pooling experts into one arena per device would win it back (open). */
size_t mat_bytes = G.wfmt==4 ? (size_t)D*Ih/2 : (size_t)D*Ih;
- size_t scl_bytes = (2*G.sc_gu+G.sc_d)/3*sizeof(float);
- G.exp_bytes = 3*dev_alloc_footprint(mat_bytes) + 3*dev_alloc_footprint(scl_bytes); /* + allocation slack */
+ G.exp_bytes = 3*dev_alloc_footprint(mat_bytes)
+ + 2*dev_alloc_footprint(G.sc_gu*sizeof(float))
+ + dev_alloc_footprint(G.sc_d*sizeof(float));
/* Per-device allowance for tier + trunk: CUDA_EXPERT_GB when numeric,
* else free minus 1 GB headroom. The heat table is loaded here too (it
diff --git a/c/tests/test_qwen36_tier_scale_budget.c b/c/tests/test_qwen36_tier_scale_budget.c
new file mode 100644
index 00000000..bb2ef81e
--- /dev/null
+++ b/c/tests/test_qwen36_tier_scale_budget.c
@@ -0,0 +1,34 @@
+/* Separate scale allocations can cross different allocator size classes. */
+#include <stdio.h>
+#include "../compat.h"
+#include "qwen36_fake_cuda.h"
+#include "../qwen36_tier.c"
+int main(void) {
+ /* D, Ih, group size (0 = int8), bytes/expert, allowance, admitted experts.
+ * Constants include three weight and three separate scale allocations. */
+ const struct { int D,Ih,gs; size_t bytes,budget; int planned; } cases[]={
+ {64,2048,0,458752,917503,1},
+ {2048,64,0,450560,901119,1},
+ {64,3000,0,655360,1310720,2},
+ {64,64,0,49152,98304,2},
+ {96,2048,64,385024,770047,1},
+ };
+ setenv("COLI_CUDA","1",1); setenv("COLI_GPUS","0",1);
+ setenv("QT_NO_WARMSTART","1",1); setenv("HEAT_FILE","",1);
+ for(size_t i=0;i<sizeof cases/sizeof *cases;i++) {
+ char gb[64];snprintf(gb,sizeof gb,"%.17g",cases[i].budget/1073741824.0);
+ setenv("CUDA_EXPERT_GB",gb,1);
+ if(!qt_init(1,3,cases[i].D,cases[i].Ih,3,1,cases[i].gs,cases[i].gs!=0)) return 1;
+ int layers[3],eids[3],n=qt_plan_fill(layers,eids,3);
+ int ok=G.exp_bytes==cases[i].bytes && n==cases[i].planned &&
+ G.used[0]==cases[i].bytes*n && G.used[0]<=G.budget[0];
+ if(!ok) fprintf(stderr,"case %zu: bytes=%zu expected=%zu planned=%d expected=%d\n",
+ i,G.exp_bytes,cases[i].bytes,n,cases[i].planned);
+ qt_shutdown();
+ /* Accommodate dev's retained host storage until teardown cleanup lands. */
+ free(G.slot);G.slot=NULL;free(G.is_x);G.is_x=NULL;
+ free(G.fill_order);G.fill_order=NULL;free(G.heat0);G.heat0=NULL;
+ if(!ok) return 1;
+ }
+ puts("tier scale budget: ok");return 0;
+}