Run frontier MoE models on hardware you already own — pure C, zero deps, experts streamed from disk. Tiny engine, immense model. 🐦
C
fix(cuda): charge the expert tier real VRAM, not logical bytes (#687)
Unknown-Findoutd9e0a67f1d46
diff --git a/c/backend_cuda.cu b/c/backend_cuda.cuindex e484cdc4..14f1b56f 100644--- a/c/backend_cuda.cu+++ b/c/backend_cuda.cu@@ -2410,6 +2410,121 @@ extern "C" size_t coli_cuda_tensor_bytes(const ColiCudaTensor *tensor) {((tensor->fmt && tensor->fmt != 6) ? tensor->scale_count * sizeof(float) : 0);}+/* What a cudaMalloc of `bytes` actually takes off the card.+ *+ * coli_cuda_tensor_bytes() above is the LOGICAL size and must stay that way -+ * it mirrors upload and free so the three cannot drift. This is a different+ * question: the allocator rounds a request up, and nothing was charging the+ * difference to the expert budget.+ *+ * Measured on an RTX 3090 (sm_86, CUDA 13.1):+ *+ * request actual overhead+ * 0.75 MiB 1.00 MiB +0.25 <- int4-g64 scale array+ * 1.00 MiB 1.00 MiB +0.00+ * 1.00 MiB + 1 B 2.00 MiB +1.00+ * 1.50 MiB 2.00 MiB +0.50+ * 2.00 MiB 2.00 MiB +0.00+ * 2.00 MiB + 1 B 4.00 MiB +2.00+ * 6.00 MiB 6.00 MiB +0.00 <- int4-g64 weight tensor+ *+ * A GLM-5.2 int4-g64 expert is three 6 MiB weight arrays, which are already on+ * a boundary and cost nothing extra, and three 0.75 MiB scale arrays, each+ * padded to 1 MiB. 3 x 0.25 = 0.75 MiB per expert unaccounted - which is the+ * 0.741 MiB/expert (sigma 0.019) measured independently on H100 and H200 in+ * #687, from the other direction.+ *+ * PROBED, not modelled. The table above is not a rule this hardcodes: the+ * rounding is a driver and architecture property, and a formula fitted to one+ * card would be silently wrong on the next. Probed once per distinct size+ * and cached, so the cost does not scale with the tier.+ *+ * cudaMemGetInfo is the obvious alternative and it does not survive contact+ * with the numbers: it is O(live allocations), measured 0.52 us empty and+ * 68.2 us with 12,000 live, so charging it per expert would be quadratic+ * across a tier that places thousands.+ *+ * Returns `bytes` unchanged if the probe cannot run. Under-reporting is the+ * pre-existing behaviour and degrades to exactly what this replaced; refusing+ * to size at all would be worse.+ */+#define COLI_CUDA_FOOTPRINT_CACHE 16+static struct { size_t req, real; } g_fp_cache[COLI_CUDA_FOOTPRINT_CACHE];+static int g_fp_cache_n = 0;++/* The probe MUST allocate several buffers and divide, not one and measure it.+ *+ * A single cudaMalloc reserves a whole 2 MiB VMM page, so one allocation of+ * anything smaller reads as a flat 2 MiB and the answer is the page size+ * rather than the per-allocation cost. Later allocations suballocate from+ * pages already reserved, so the real amortised figure only appears once+ * enough of them are live to fill a page. Measured both ways on sm_86, same+ * 786,432-byte request:+ *+ * 1 allocation 2.00 MiB <- the page, not the allocation+ * 64 allocations 1.00 MiB <- the truth, and what the tier will pay+ *+ * The single-shot version of this function shipped in an earlier draft and its+ * own test caught it: it over-reported every expert by 3x and would have+ * shrunk the tier far more than the bug it fixes.+ */+#define COLI_CUDA_FOOTPRINT_PROBE_BYTES (24u << 20) /* transient probe budget */+#define COLI_CUDA_FOOTPRINT_PROBE_MAX 64++extern "C" size_t coli_cuda_alloc_footprint(size_t bytes) {+ if (!bytes) return 0;+ for (int i = 0; i < g_fp_cache_n; i++)+ if (g_fp_cache[i].req == bytes) return g_fp_cache[i].real;++ /* Enough allocations to cross a page boundary, capped so the probe never+ * takes a meaningful bite out of a card that is about to be filled. */+ int want = (int)(COLI_CUDA_FOOTPRINT_PROBE_BYTES / bytes);+ if (want < 8) want = 8;+ if (want > COLI_CUDA_FOOTPRINT_PROBE_MAX) want = COLI_CUDA_FOOTPRINT_PROBE_MAX;++ size_t free_before = 0, free_after = 0, total = 0, real = bytes;+ void *p[COLI_CUDA_FOOTPRINT_PROBE_MAX];+ int made = 0;+ if (cudaMemGetInfo(&free_before, &total) == cudaSuccess) {+ for (int i = 0; i < want; i++) {+ if (cudaMalloc(&p[i], bytes) != cudaSuccess) break;+ made++;+ }+ if (made > 0 && cudaMemGetInfo(&free_after, &total) == cudaSuccess &&+ free_before > free_after) {+ size_t avg = (free_before - free_after) / (size_t)made;+ /* Only ever round UP. A concurrent free elsewhere on the device+ * can make the delta read small, and charging less than the+ * logical size is the failure this exists to fix. */+ if (avg > real) real = avg;+ }+ for (int i = 0; i < made; i++) cudaFree(p[i]);+ }+ /* Cache even a failed probe: `real` is then the logical size, which is the+ * pre-existing behaviour, and retrying per expert on a card that just+ * refused 8 allocations would be the worst possible time to try again. */+ if (g_fp_cache_n < COLI_CUDA_FOOTPRINT_CACHE)+ g_fp_cache[g_fp_cache_n++] = { bytes, real };+ return real;+}++/* Same split as coli_cuda_tensor_bytes, but per ALLOCATION rather than summed+ * first: the weights and the scales are two separate cudaMallocs and each is+ * rounded on its own. Summing then rounding once would miss the scale array's+ * padding entirely, which is the whole term. */+extern "C" size_t coli_cuda_tensor_vram(const ColiCudaTensor *tensor) {+ if (!tensor) return 0;+ size_t storage_bytes =+#ifdef COLI_ANS+ tensor->compressed ? tensor->archive_bytes :+#endif+ tensor->weight_bytes;+ size_t total = coli_cuda_alloc_footprint(storage_bytes);+ if (tensor->fmt && tensor->fmt != 6)+ total += coli_cuda_alloc_footprint(tensor->scale_count * sizeof(float));+ return total;+}+extern "C" int coli_cuda_tensor_device(const ColiCudaTensor *tensor) {return tensor ? tensor->device : -1;}diff --git a/c/backend_cuda.h b/c/backend_cuda.hindex 597f4273..953e8b39 100644--- a/c/backend_cuda.h+++ b/c/backend_cuda.h@@ -187,6 +187,8 @@ COLI_CUDA_DLLEXPORT int coli_cuda_attention_project_ragged(ColiCudaTensor *kv_b,COLI_CUDA_DLLEXPORT void coli_cuda_tensor_free(ColiCudaTensor *tensor);COLI_CUDA_DLLEXPORT size_t coli_cuda_tensor_bytes(const ColiCudaTensor *tensor);+COLI_CUDA_DLLEXPORT size_t coli_cuda_alloc_footprint(size_t bytes);+COLI_CUDA_DLLEXPORT size_t coli_cuda_tensor_vram(const ColiCudaTensor *tensor);COLI_CUDA_DLLEXPORT int coli_cuda_tensor_device(const ColiCudaTensor *tensor);/* Replace a resident tensor's contents without reallocating its device slot. */diff --git a/c/colibri.c b/c/colibri.cindex da06e84e..a566b453 100644--- a/c/colibri.c+++ b/c/colibri.c@@ -9814,11 +9814,31 @@ static void pin_load(Model *m, const char *statspath, double gb, int trusted){}#endifif(uploaded){+ /* VRAM, not logical bytes (#687). The allocator rounds+ * every cudaMalloc up and nothing was charging the+ * difference, so `remaining` drifted optimistic by a+ * term that GREW with the tier: an int4-g64 scale array+ * is 0.75 MiB and lands in 1 MiB, three per expert, so+ * 0.75 MiB per expert uncounted (measured on sm_86;+ * #687 measured 0.741 +/- 0.019 on H100/H200 from the+ * other direction). At 6,235 experts that is 4.6 GB+ * against a flat 2 GB reserve, which is why auto could+ * claim the card to within 4 MiB and then fail every+ * lazy dense upload afterwards.+ *+ * m->gpu_expert_bytes stays LOGICAL: it is reported as+ * the tier's size and compared against `budget`, and+ * quoting padding to the user as model bytes would+ * trade one wrong number for another. */int64_t actual=(int64_t)coli_cuda_tensor_bytes(s->g.cuda)+(int64_t)coli_cuda_tensor_bytes(s->u.cuda)+(int64_t)coli_cuda_tensor_bytes(s->d.cuda);+ int64_t vram =(int64_t)coli_cuda_tensor_vram(s->g.cuda)+ +(int64_t)coli_cuda_tensor_vram(s->u.cuda)+ +(int64_t)coli_cuda_tensor_vram(s->d.cuda);+ if(vram<actual) vram=actual;m->gpu_expert_count++; m->gpu_expert_bytes+=actual;- remaining[best]-=actual; placed_b[best]+=actual; placed_n[best]++;+ remaining[best]-=vram; placed_b[best]+=actual; placed_n[best]++;placed_w[best]+=(double)r[a].c;if(g_cuda_release_host){ expert_host_release(m,s); pin_host_released+=(double)need; }placed=1;