Run frontier MoE models on hardware you already own — pure C, zero deps, experts streamed from disk. Tiny engine, immense model. 🐦
C
cuda: fix resident MXFP4 scale layout and accounting
ZacharyZcRa65ea155d85a
diff --git a/c/backend_cuda.cu b/c/backend_cuda.cuindex 07d5dfc4..ad205bd8 100644--- a/c/backend_cuda.cu+++ b/c/backend_cuda.cu@@ -63,7 +63,7 @@ struct ColiCudaTensor {int fmt, I, O, device;int gs; /* quant group size; 0 = per-row scales (#334) */int ng; /* number of scale groups per row = ceil(I/gs) for fmt=4 */- size_t scale_count; /* floats in `scales`: O per-row, O*ng grouped */+ size_t scale_count; /* scale elements: ue8m0 bytes for fmt=7, floats otherwise */int tracked;int weights_owned;#ifdef COLI_ANS@@ -74,6 +74,11 @@ struct ColiCudaTensor {int ragged_count;};+static size_t tensor_scale_bytes(const ColiCudaTensor *t) {+ if (!t->fmt || t->fmt == 6) return 0;+ return t->scale_count * (t->fmt == 7 ? sizeof(uint8_t) : sizeof(float));+}+#ifdef COLI_ANSstruct AnsArenaChunk { uint8_t *p; size_t used,cap; };#endif@@ -1419,6 +1424,10 @@ extern "C" int coli_cuda_tensor_upload(ColiCudaTensor **tensor,t->gs = (fmt==4 && g_upload_gs>0) ? g_upload_gs : 0;t->ng = t->gs ? (I + t->gs - 1) / t->gs : 1;t->scale_count = t->gs ? (size_t)O * (size_t)t->ng : (size_t)O;+ if (fmt == 7) {+ t->ng = (I + 31) / 32;+ t->scale_count = (size_t)O * t->ng;+ }if (fmt == 8) { /* per-128x128-block scales: [ceil(O/128), ceil(I/128)] */t->ng = (I + 127) / 128;t->scale_count = (size_t)((O + 127) / 128) * (size_t)t->ng;@@ -1439,8 +1448,8 @@ extern "C" int coli_cuda_tensor_upload(ColiCudaTensor **tensor,offset_to_signed_s4<<<(unsigned)((t->weight_bytes+255)/256),256>>>((uint8_t*)t->weights,t->weight_bytes);if(!cuda_ok(cudaGetLastError(),"int4 weight conversion")){coli_cuda_tensor_free(t);return 0;}}if (fmt && fmt != 6) {- if (!cuda_ok(cudaMalloc(&t->scales, t->scale_count * sizeof(float)), "scale allocation") ||- !cuda_ok(cudaMemcpy(t->scales, scales, t->scale_count * sizeof(float), cudaMemcpyHostToDevice), "scale upload")) {+ if (!cuda_ok(cudaMalloc(&t->scales, tensor_scale_bytes(t)), "scale allocation") ||+ !cuda_ok(cudaMemcpy(t->scales, scales, tensor_scale_bytes(t), cudaMemcpyHostToDevice), "scale upload")) {coli_cuda_tensor_free(t);return 0;}@@ -1448,7 +1457,7 @@ extern "C" int coli_cuda_tensor_upload(ColiCudaTensor **tensor,if (fmt == 6) t->scale_count = 0; /* in-block scales: nothing separate to track */t->tracked = 1;ctx->tensor_count++;- ctx->tensor_bytes += t->weight_bytes + ((fmt && fmt != 6) ? t->scale_count * sizeof(float) : 0);+ ctx->tensor_bytes += t->weight_bytes + tensor_scale_bytes(t);*tensor = t;return 1;}@@ -1634,10 +1643,9 @@ extern "C" int coli_cuda_tensor_update(ColiCudaTensor *tensor,(uint8_t*)tensor->weights,tensor->weight_bytes);if(!cuda_ok(cudaGetLastError(),"int4 weight refresh")) return 0;}- /* fmt=6 has no scale buffer at all (scales live in-block, scale_count 0), and- * the fallback below would otherwise copy O floats out of a NULL host pointer. */+ /* fmt=6 stores scales in-block; fmt=7 stores byte exponents separately. */return !tensor->fmt || tensor->fmt==6 || cuda_ok(cudaMemcpy(tensor->scales,scales,- (tensor->scale_count?tensor->scale_count:(size_t)tensor->O)*sizeof(float),+ tensor_scale_bytes(tensor),cudaMemcpyHostToDevice),"scale refresh");}@@ -2386,19 +2394,14 @@ extern "C" void coli_cuda_tensor_free(ColiCudaTensor *tensor) {DeviceContext *ctx = find_ctx(tensor->device);if (ctx) select_ctx(ctx);if (tensor->tracked && ctx) {- /* Must mirror the upload's accounting exactly -- literally the same- * expression upload uses to charge (scale_count * sizeof(float), gated- * on fmt=6 never having a separate scale buffer), so the two can no- * longer drift independently. Over-subtracting here trips the >= guard- * below, which silently leaves the tensor's bytes on the device counter- * forever. */+ /* Charge and release the same format-specific scale storage. */size_t storage_bytes =#ifdef COLI_ANStensor->compressed ? tensor->archive_bytes :#endiftensor->weight_bytes;size_t bytes = storage_bytes +- ((tensor->fmt && tensor->fmt != 6) ? tensor->scale_count * sizeof(float) : 0);+ tensor_scale_bytes(tensor);if (ctx->tensor_count) ctx->tensor_count--;if (ctx->tensor_bytes >= bytes) ctx->tensor_bytes -= bytes;}@@ -2410,19 +2413,14 @@ extern "C" void coli_cuda_tensor_free(ColiCudaTensor *tensor) {extern "C" size_t coli_cuda_tensor_bytes(const ColiCudaTensor *tensor) {if (!tensor) return 0;- /* Must mirror upload's and free's accounting exactly -- literally the same- * expression they use (scale_count * sizeof(float), gated on fmt=6 never- * having a separate scale buffer) -- so all three can no longer drift- * independently. The prior `O * ng` shape over-reported for fmt=8 (real- * footprint is (O+127)/128 * ng block scales, not O * ng) and for fmt=6- * (which has no separate scale buffer at all). */+ /* Logical size uses the same scale layout as upload and free. */size_t storage_bytes =#ifdef COLI_ANStensor->compressed ? tensor->archive_bytes :#endiftensor->weight_bytes;return storage_bytes +- ((tensor->fmt && tensor->fmt != 6) ? tensor->scale_count * sizeof(float) : 0);+ tensor_scale_bytes(tensor);}/* What a cudaMalloc of `bytes` actually takes off the card.@@ -2536,7 +2534,7 @@ extern "C" size_t coli_cuda_tensor_vram(const ColiCudaTensor *tensor) {tensor->weight_bytes;size_t total = coli_cuda_alloc_footprint(storage_bytes);if (tensor->fmt && tensor->fmt != 6)- total += coli_cuda_alloc_footprint(tensor->scale_count * sizeof(float));+ total += coli_cuda_alloc_footprint(tensor_scale_bytes(tensor));return total;}diff --git a/c/tests/test_mxfp4_cuda.cu b/c/tests/test_mxfp4_cuda.cuindex 6604560d..b329ea40 100644--- a/c/tests/test_mxfp4_cuda.cu+++ b/c/tests/test_mxfp4_cuda.cu@@ -204,6 +204,41 @@ static void all_codes(void) {else printf(" ok all 16 e2m1 codes decode exactly (cpu == gpu == spec)\n");}+/* Resident upload/update must use O*ceil(I/32) BYTES, including tails.+ * Positive inputs avoid cancellation so every missing group affects the result. */+static void resident_case(int I, int O) {+ const int S = 2, rb = (I + 1) / 2, ng = (I + 31) / 32;+ uint8_t *q = (uint8_t *)malloc((size_t)O * rb);+ uint8_t *sc = (uint8_t *)malloc((size_t)O * ng);+ float *x = (float *)malloc((size_t)S * I * sizeof(float));+ float *want = (float *)malloc((size_t)S * O * sizeof(float));+ float *got = (float *)malloc((size_t)S * O * sizeof(float));+ memset(q, 0x22, (size_t)O * rb);+ for (int i = 0; i < S * I; i++) x[i] = 0.25f * (1 + i % 3);+ size_t count0, bytes0, count1, bytes1;+ coli_cuda_stats(0, &count0, &bytes0);+ ColiCudaTensor *t = nullptr;+ for (int pass = 0; pass < 2; pass++) {+ for (int i = 0; i < O * ng; i++) sc[i] = (uint8_t)(125 + (i + pass) % 5);+ const float *scales = reinterpret_cast<const float *>(sc);+ int ok = pass ? coli_cuda_tensor_update(t, q, scales)+ : coli_cuda_tensor_upload(&t, q, scales, 7, I, O, 0);+ if (!ok) { printf(" FAIL resident upload/update\n"); fails++; break; }+ size_t expected = (size_t)O * (rb + ng);+ coli_cuda_stats(0, &count1, &bytes1);+ if (coli_cuda_tensor_bytes(t) != expected || count1 != count0 + 1 || bytes1 != bytes0 + expected) {+ printf(" FAIL resident byte accounting I=%d O=%d\n", I, O); fails++;+ }+ mxfp4_ref(want, x, q, sc, S, I, O);+ if (!coli_cuda_matmul(&t, got, x, nullptr, nullptr, 7, S, I, O, 0, 0)) fails++;+ else compare_case(pass ? "resident refresh" : "resident upload", want, got, S, I, O);+ }+ coli_cuda_tensor_free(t);+ coli_cuda_stats(0, &count1, &bytes1);+ if (count0 != count1 || bytes0 != bytes1) { printf(" FAIL resident free accounting\n"); fails++; }+ free(q); free(sc); free(x); free(want); free(got);+}+int main(void) {int ndev = 0;if (cudaGetDeviceCount(&ndev) != cudaSuccess || ndev < 1) {@@ -216,6 +251,8 @@ int main(void) {return 0;}+ resident_case(33, 3); /* 6 exponent bytes, not 12 */+ resident_case(257, 5); /* 45 exponent bytes, not 20 */all_codes();one_case("decode + matmul", 1, 64, 32, -1);one_case("multi-row batch", 4, 128, 64, -1);