Skip to content

Run frontier MoE models on hardware you already own — pure C, zero deps, experts streamed from disk. Tiny engine, immense model. 🐦

C

tier: charge VRAM at cudaMalloc granularity; accept COLI_GPU

kreuzzelgd0a382dfbdee
diff --git a/c/qwen36_tier.c b/c/qwen36_tier.c
index 2f4fde7d..81299801 100644
--- a/c/qwen36_tier.c
+++ b/c/qwen36_tier.c
@@ -401,6 +401,15 @@ int qt_init_fp8(int nl, int ne, int D, int Ih, int cap, int topk, const float *e
return ok;
}
+/* VRAM an allocation of `bytes` really occupies (cudaMalloc granularity,
+ * see the exp_bytes comment in qt_init). */
+static size_t dev_alloc_footprint(size_t bytes){
+ const size_t MiB = 1048576u;
+ if(bytes <= 10240u) return 10240u;
+ if(bytes <= MiB) return MiB;
+ return (bytes + 2*MiB - 1) / (2*MiB) * (2*MiB);
+}
+
int qt_init(int nl, int ne, int D, int Ih, int cap, int topk, int expert_gs,
int expert_is_int4){
const char *e=getenv("COLI_CUDA");
@@ -418,8 +427,12 @@ int qt_init(int nl, int ne, int D, int Ih, int cap, int topk, int expert_gs,
* would otherwise stand in for the current environment. */
G_place_done = 0; G_place_n = 0; G_auto_on = 0;
- /* devices: COLI_GPUS="0,1" (default: first two visible devices) */
+ /* devices: COLI_GPUS="0,1" (default: first two visible devices).
+ * COLI_GPU is the singular the planner writes for a one-device plan
+ * (resource_plan.py) and colibri.c reads; accept it here as well, or a
+ * `coli chat --gpu 1` lands on every visible device. */
const char *gl=getenv("COLI_GPUS");
+ if(!gl || !*gl) gl=getenv("COLI_GPU");
if (gl && *gl) {
char buf[128]; snprintf(buf,sizeof buf,"%s",gl);
for(char *t=strtok(buf,","); t && G.ndev<QT_MAX_DEV; t=strtok(NULL,","))
@@ -475,8 +488,20 @@ int qt_init(int nl, int ne, int D, int Ih, int cap, int topk, int expert_gs,
G.sc_gu = expert_gs ? (size_t)Ih * ((D + expert_gs - 1)/expert_gs) : (size_t)Ih;
G.sc_d = expert_gs ? (size_t)D * ((Ih + expert_gs - 1)/expert_gs) : (size_t)D;
}
- G.exp_bytes = (G.wfmt==4 ? 3ull*D*Ih/2 : 3ull*D*Ih)
- + (2*G.sc_gu+G.sc_d)*sizeof(float) + 4096; /* + allocation slack */
+ /* Charge what the device allocator takes, not what the bytes measure:
+ * cudaMalloc rounds an allocation above 1 MiB up to a multiple of 2 MiB
+ * and one at or below 1 MiB up to 1 MiB (measured with cudaMemGetInfo,
+ * driver 5xx; 1,638,400 B -> 2 MiB, 819,200 B -> 1 MiB, 400 B -> 10 KiB).
+ * An expert is three weight allocations plus three scale allocations.
+ * Charged by payload, the fp8 Qwen3.8 expert (3 x 1.56 MiB) looked like
+ * 4.69 MiB and took 6.03 MiB: the budget filled the card to the last
+ * megabyte and the uploader ran into "tensor allocation: out of memory"
+ * before its stop-trying fallback shrank the budget. Now the planned count
+ * is the resident count. The 22-28 % the granularity costs is real; only
+ * pooling experts into one arena per device would win it back (open). */
+ size_t mat_bytes = G.wfmt==4 ? (size_t)D*Ih/2 : (size_t)D*Ih;
+ size_t scl_bytes = (2*G.sc_gu+G.sc_d)/3*sizeof(float);
+ G.exp_bytes = 3*dev_alloc_footprint(mat_bytes) + 3*dev_alloc_footprint(scl_bytes); /* + allocation slack */
/* Per-device allowance for tier + trunk: CUDA_EXPERT_GB when numeric,
* else free minus 1 GB headroom. The heat table is loaded here too (it
diff --git a/c/tests/test_qwen36_tier_autoplace.c b/c/tests/test_qwen36_tier_autoplace.c
index 14731b19..3c4c4041 100644
--- a/c/tests/test_qwen36_tier_autoplace.c
+++ b/c/tests/test_qwen36_tier_autoplace.c
@@ -92,8 +92,10 @@ int main(void) {
enum { NL = 1, NE = 48 };
const char *heat = write_uniform_heat(NL, NE);
char gb[64];
- /* exp_bytes for D=64, IH=32, per-row int4: 3072 + 512 + 4096 = 7680 */
- size_t exp_bytes = 3ull * D * IH / 2 + (2 * IH + D) * sizeof(float) + 4096;
+ /* exp_bytes for D=64, IH=32, per-row int4, at cudaMalloc granularity:
+ * three 1 KiB matrices charged 1 MiB each, three scale tables 10 KiB each */
+ size_t exp_bytes = 3 * dev_alloc_footprint((size_t)D * IH / 2)
+ + 3 * dev_alloc_footprint((2 * IH + D) / 3 * sizeof(float));
snprintf(gb, sizeof gb, "%.15f", (double)(16 * exp_bytes + exp_bytes / 2) / 1073741824.0);
/* the allowance as the tier will parse it back from the string */
size_t allowance = (size_t)(atof(gb) * 1024.0 * 1024.0 * 1024.0);
diff --git a/c/tests/test_qwen36_tier_fp8.c b/c/tests/test_qwen36_tier_fp8.c
index b17aa7fe..5abc8723 100644
--- a/c/tests/test_qwen36_tier_fp8.c
+++ b/c/tests/test_qwen36_tier_fp8.c
@@ -66,14 +66,14 @@ int main(void) {
/* ---- 1. the mode starts with cap < n_experts, publishes the LUT, sizes fmt 8 ---- */
printf(" 1. init\n");
/* allowance for exactly 3 experts on the one device */
- size_t exp_bytes = 3ull * D * IH + 3ull * NSC * sizeof(float) + 4096;
+ size_t exp_bytes = 3 * dev_alloc_footprint((size_t)D * IH) + 3 * dev_alloc_footprint(NSC * sizeof(float));
char gb[64]; snprintf(gb, sizeof gb, "%.15f", (double)(3 * exp_bytes + exp_bytes / 2) / 1073741824.0);
setenv("CUDA_EXPERT_GB", gb, 1);
fake_ndev = 1; fake_uploads = 0; fake_lut_published = 0;
check(qt_init_fp8(NL, NE, D, IH, CAP, TOPK, lut), "fp8 streaming mode must start with cap 4 < 16 experts");
check(G.wfmt == 8, "weight format is 8");
check(fake_lut_published, "e4m3 LUT published to the backend before any upload");
- check(G.exp_bytes == exp_bytes, "exp_bytes = 3*D*IH bytes + 3 block-scale tables + slack");
+ check(G.exp_bytes == exp_bytes, "exp_bytes = three fmt-8 matrices + three scale tables at cudaMalloc granularity");
check(G.sc_gu == NSC && G.sc_d == NSC, "one scale per 128x128 block per matrix");
/* ---- 2. a note uploads fmt 8, bytes intact, scales in order, pointers dropped ---- */
diff --git a/docs/qwen36-cuda-tier.md b/docs/qwen36-cuda-tier.md
index 9fd5f30b..a0f05c7b 100644
--- a/docs/qwen36-cuda-tier.md
+++ b/docs/qwen36-cuda-tier.md
@@ -6,6 +6,15 @@ in RAM, the **hot** ones are promoted into DEVICE_LOCAL VRAM across one or
more GPUs and computed there through the existing shared CUDA backend
(`backend_cuda.cu` expert-group API — no new backend).
+> **A second mode.** Besides the Qwen3.6 modes below, the tier has an *fp8
+> streaming mode* (`qt_init_fp8`) for an engine whose experts do not all live
+> in RAM: `cap` may be smaller than the expert count, the tier copies each
+> expert's e4m3 slab and block scales when the engine reports it and keeps no
+> pointer into the engine's slot, and promotion happens at report time instead
+> of a warmstart. It is exercised by `tests/test_qwen36_tier_fp8.c`; the
+> Qwen3.8 engine that uses it comes in its own PR. Everything below describes
+> the Qwen3.6 modes unless it says otherwise.
+
## How it works
- **Home device:** expert `eid` lives on GPU `eid % n_gpus`; no duplicates.