Skip to content

Run frontier MoE models on hardware you already own — pure C, zero deps, experts streamed from disk. Tiny engine, immense model. 🐦

C

fix(cuda): charge the expert tier real VRAM, not logical bytes (#687)

Unknown-Findoutd9e0a67f1d46
diff --git a/c/backend_cuda.cu b/c/backend_cuda.cu
index e484cdc4..14f1b56f 100644
--- a/c/backend_cuda.cu
+++ b/c/backend_cuda.cu
@@ -2410,6 +2410,121 @@ extern "C" size_t coli_cuda_tensor_bytes(const ColiCudaTensor *tensor) {
((tensor->fmt && tensor->fmt != 6) ? tensor->scale_count * sizeof(float) : 0);
}
+/* What a cudaMalloc of `bytes` actually takes off the card.
+ *
+ * coli_cuda_tensor_bytes() above is the LOGICAL size and must stay that way -
+ * it mirrors upload and free so the three cannot drift. This is a different
+ * question: the allocator rounds a request up, and nothing was charging the
+ * difference to the expert budget.
+ *
+ * Measured on an RTX 3090 (sm_86, CUDA 13.1):
+ *
+ * request actual overhead
+ * 0.75 MiB 1.00 MiB +0.25 <- int4-g64 scale array
+ * 1.00 MiB 1.00 MiB +0.00
+ * 1.00 MiB + 1 B 2.00 MiB +1.00
+ * 1.50 MiB 2.00 MiB +0.50
+ * 2.00 MiB 2.00 MiB +0.00
+ * 2.00 MiB + 1 B 4.00 MiB +2.00
+ * 6.00 MiB 6.00 MiB +0.00 <- int4-g64 weight tensor
+ *
+ * A GLM-5.2 int4-g64 expert is three 6 MiB weight arrays, which are already on
+ * a boundary and cost nothing extra, and three 0.75 MiB scale arrays, each
+ * padded to 1 MiB. 3 x 0.25 = 0.75 MiB per expert unaccounted - which is the
+ * 0.741 MiB/expert (sigma 0.019) measured independently on H100 and H200 in
+ * #687, from the other direction.
+ *
+ * PROBED, not modelled. The table above is not a rule this hardcodes: the
+ * rounding is a driver and architecture property, and a formula fitted to one
+ * card would be silently wrong on the next. Probed once per distinct size
+ * and cached, so the cost does not scale with the tier.
+ *
+ * cudaMemGetInfo is the obvious alternative and it does not survive contact
+ * with the numbers: it is O(live allocations), measured 0.52 us empty and
+ * 68.2 us with 12,000 live, so charging it per expert would be quadratic
+ * across a tier that places thousands.
+ *
+ * Returns `bytes` unchanged if the probe cannot run. Under-reporting is the
+ * pre-existing behaviour and degrades to exactly what this replaced; refusing
+ * to size at all would be worse.
+ */
+#define COLI_CUDA_FOOTPRINT_CACHE 16
+static struct { size_t req, real; } g_fp_cache[COLI_CUDA_FOOTPRINT_CACHE];
+static int g_fp_cache_n = 0;
+
+/* The probe MUST allocate several buffers and divide, not one and measure it.
+ *
+ * A single cudaMalloc reserves a whole 2 MiB VMM page, so one allocation of
+ * anything smaller reads as a flat 2 MiB and the answer is the page size
+ * rather than the per-allocation cost. Later allocations suballocate from
+ * pages already reserved, so the real amortised figure only appears once
+ * enough of them are live to fill a page. Measured both ways on sm_86, same
+ * 786,432-byte request:
+ *
+ * 1 allocation 2.00 MiB <- the page, not the allocation
+ * 64 allocations 1.00 MiB <- the truth, and what the tier will pay
+ *
+ * The single-shot version of this function shipped in an earlier draft and its
+ * own test caught it: it over-reported every expert by 3x and would have
+ * shrunk the tier far more than the bug it fixes.
+ */
+#define COLI_CUDA_FOOTPRINT_PROBE_BYTES (24u << 20) /* transient probe budget */
+#define COLI_CUDA_FOOTPRINT_PROBE_MAX 64
+
+extern "C" size_t coli_cuda_alloc_footprint(size_t bytes) {
+ if (!bytes) return 0;
+ for (int i = 0; i < g_fp_cache_n; i++)
+ if (g_fp_cache[i].req == bytes) return g_fp_cache[i].real;
+
+ /* Enough allocations to cross a page boundary, capped so the probe never
+ * takes a meaningful bite out of a card that is about to be filled. */
+ int want = (int)(COLI_CUDA_FOOTPRINT_PROBE_BYTES / bytes);
+ if (want < 8) want = 8;
+ if (want > COLI_CUDA_FOOTPRINT_PROBE_MAX) want = COLI_CUDA_FOOTPRINT_PROBE_MAX;
+
+ size_t free_before = 0, free_after = 0, total = 0, real = bytes;
+ void *p[COLI_CUDA_FOOTPRINT_PROBE_MAX];
+ int made = 0;
+ if (cudaMemGetInfo(&free_before, &total) == cudaSuccess) {
+ for (int i = 0; i < want; i++) {
+ if (cudaMalloc(&p[i], bytes) != cudaSuccess) break;
+ made++;
+ }
+ if (made > 0 && cudaMemGetInfo(&free_after, &total) == cudaSuccess &&
+ free_before > free_after) {
+ size_t avg = (free_before - free_after) / (size_t)made;
+ /* Only ever round UP. A concurrent free elsewhere on the device
+ * can make the delta read small, and charging less than the
+ * logical size is the failure this exists to fix. */
+ if (avg > real) real = avg;
+ }
+ for (int i = 0; i < made; i++) cudaFree(p[i]);
+ }
+ /* Cache even a failed probe: `real` is then the logical size, which is the
+ * pre-existing behaviour, and retrying per expert on a card that just
+ * refused 8 allocations would be the worst possible time to try again. */
+ if (g_fp_cache_n < COLI_CUDA_FOOTPRINT_CACHE)
+ g_fp_cache[g_fp_cache_n++] = { bytes, real };
+ return real;
+}
+
+/* Same split as coli_cuda_tensor_bytes, but per ALLOCATION rather than summed
+ * first: the weights and the scales are two separate cudaMallocs and each is
+ * rounded on its own. Summing then rounding once would miss the scale array's
+ * padding entirely, which is the whole term. */
+extern "C" size_t coli_cuda_tensor_vram(const ColiCudaTensor *tensor) {
+ if (!tensor) return 0;
+ size_t storage_bytes =
+#ifdef COLI_ANS
+ tensor->compressed ? tensor->archive_bytes :
+#endif
+ tensor->weight_bytes;
+ size_t total = coli_cuda_alloc_footprint(storage_bytes);
+ if (tensor->fmt && tensor->fmt != 6)
+ total += coli_cuda_alloc_footprint(tensor->scale_count * sizeof(float));
+ return total;
+}
+
extern "C" int coli_cuda_tensor_device(const ColiCudaTensor *tensor) {
return tensor ? tensor->device : -1;
}
diff --git a/c/backend_cuda.h b/c/backend_cuda.h
index 597f4273..953e8b39 100644
--- a/c/backend_cuda.h
+++ b/c/backend_cuda.h
@@ -187,6 +187,8 @@ COLI_CUDA_DLLEXPORT int coli_cuda_attention_project_ragged(ColiCudaTensor *kv_b,
COLI_CUDA_DLLEXPORT void coli_cuda_tensor_free(ColiCudaTensor *tensor);
COLI_CUDA_DLLEXPORT size_t coli_cuda_tensor_bytes(const ColiCudaTensor *tensor);
+COLI_CUDA_DLLEXPORT size_t coli_cuda_alloc_footprint(size_t bytes);
+COLI_CUDA_DLLEXPORT size_t coli_cuda_tensor_vram(const ColiCudaTensor *tensor);
COLI_CUDA_DLLEXPORT int coli_cuda_tensor_device(const ColiCudaTensor *tensor);
/* Replace a resident tensor's contents without reallocating its device slot. */
diff --git a/c/colibri.c b/c/colibri.c
index da06e84e..a566b453 100644
--- a/c/colibri.c
+++ b/c/colibri.c
@@ -9814,11 +9814,31 @@ static void pin_load(Model *m, const char *statspath, double gb, int trusted){
}
#endif
if(uploaded){
+ /* VRAM, not logical bytes (#687). The allocator rounds
+ * every cudaMalloc up and nothing was charging the
+ * difference, so `remaining` drifted optimistic by a
+ * term that GREW with the tier: an int4-g64 scale array
+ * is 0.75 MiB and lands in 1 MiB, three per expert, so
+ * 0.75 MiB per expert uncounted (measured on sm_86;
+ * #687 measured 0.741 +/- 0.019 on H100/H200 from the
+ * other direction). At 6,235 experts that is 4.6 GB
+ * against a flat 2 GB reserve, which is why auto could
+ * claim the card to within 4 MiB and then fail every
+ * lazy dense upload afterwards.
+ *
+ * m->gpu_expert_bytes stays LOGICAL: it is reported as
+ * the tier's size and compared against `budget`, and
+ * quoting padding to the user as model bytes would
+ * trade one wrong number for another. */
int64_t actual=(int64_t)coli_cuda_tensor_bytes(s->g.cuda)
+(int64_t)coli_cuda_tensor_bytes(s->u.cuda)
+(int64_t)coli_cuda_tensor_bytes(s->d.cuda);
+ int64_t vram =(int64_t)coli_cuda_tensor_vram(s->g.cuda)
+ +(int64_t)coli_cuda_tensor_vram(s->u.cuda)
+ +(int64_t)coli_cuda_tensor_vram(s->d.cuda);
+ if(vram<actual) vram=actual;
m->gpu_expert_count++; m->gpu_expert_bytes+=actual;
- remaining[best]-=actual; placed_b[best]+=actual; placed_n[best]++;
+ remaining[best]-=vram; placed_b[best]+=actual; placed_n[best]++;
placed_w[best]+=(double)r[a].c;
if(g_cuda_release_host){ expert_host_release(m,s); pin_host_released+=(double)need; }
placed=1;