Skip to content

Run frontier MoE models on hardware you already own — pure C, zero deps, experts streamed from disk. Tiny engine, immense model. 🐦

C

cuda: fix resident MXFP4 scale layout and accounting

ZacharyZcRa65ea155d85a
diff --git a/c/backend_cuda.cu b/c/backend_cuda.cu
index 07d5dfc4..ad205bd8 100644
--- a/c/backend_cuda.cu
+++ b/c/backend_cuda.cu
@@ -63,7 +63,7 @@ struct ColiCudaTensor {
int fmt, I, O, device;
int gs; /* quant group size; 0 = per-row scales (#334) */
int ng; /* number of scale groups per row = ceil(I/gs) for fmt=4 */
- size_t scale_count; /* floats in `scales`: O per-row, O*ng grouped */
+ size_t scale_count; /* scale elements: ue8m0 bytes for fmt=7, floats otherwise */
int tracked;
int weights_owned;
#ifdef COLI_ANS
@@ -74,6 +74,11 @@ struct ColiCudaTensor {
int ragged_count;
};
+static size_t tensor_scale_bytes(const ColiCudaTensor *t) {
+ if (!t->fmt || t->fmt == 6) return 0;
+ return t->scale_count * (t->fmt == 7 ? sizeof(uint8_t) : sizeof(float));
+}
+
#ifdef COLI_ANS
struct AnsArenaChunk { uint8_t *p; size_t used,cap; };
#endif
@@ -1419,6 +1424,10 @@ extern "C" int coli_cuda_tensor_upload(ColiCudaTensor **tensor,
t->gs = (fmt==4 && g_upload_gs>0) ? g_upload_gs : 0;
t->ng = t->gs ? (I + t->gs - 1) / t->gs : 1;
t->scale_count = t->gs ? (size_t)O * (size_t)t->ng : (size_t)O;
+ if (fmt == 7) {
+ t->ng = (I + 31) / 32;
+ t->scale_count = (size_t)O * t->ng;
+ }
if (fmt == 8) { /* per-128x128-block scales: [ceil(O/128), ceil(I/128)] */
t->ng = (I + 127) / 128;
t->scale_count = (size_t)((O + 127) / 128) * (size_t)t->ng;
@@ -1439,8 +1448,8 @@ extern "C" int coli_cuda_tensor_upload(ColiCudaTensor **tensor,
offset_to_signed_s4<<<(unsigned)((t->weight_bytes+255)/256),256>>>((uint8_t*)t->weights,t->weight_bytes);
if(!cuda_ok(cudaGetLastError(),"int4 weight conversion")){coli_cuda_tensor_free(t);return 0;}}
if (fmt && fmt != 6) {
- if (!cuda_ok(cudaMalloc(&t->scales, t->scale_count * sizeof(float)), "scale allocation") ||
- !cuda_ok(cudaMemcpy(t->scales, scales, t->scale_count * sizeof(float), cudaMemcpyHostToDevice), "scale upload")) {
+ if (!cuda_ok(cudaMalloc(&t->scales, tensor_scale_bytes(t)), "scale allocation") ||
+ !cuda_ok(cudaMemcpy(t->scales, scales, tensor_scale_bytes(t), cudaMemcpyHostToDevice), "scale upload")) {
coli_cuda_tensor_free(t);
return 0;
}
@@ -1448,7 +1457,7 @@ extern "C" int coli_cuda_tensor_upload(ColiCudaTensor **tensor,
if (fmt == 6) t->scale_count = 0; /* in-block scales: nothing separate to track */
t->tracked = 1;
ctx->tensor_count++;
- ctx->tensor_bytes += t->weight_bytes + ((fmt && fmt != 6) ? t->scale_count * sizeof(float) : 0);
+ ctx->tensor_bytes += t->weight_bytes + tensor_scale_bytes(t);
*tensor = t;
return 1;
}
@@ -1634,10 +1643,9 @@ extern "C" int coli_cuda_tensor_update(ColiCudaTensor *tensor,
(uint8_t*)tensor->weights,tensor->weight_bytes);
if(!cuda_ok(cudaGetLastError(),"int4 weight refresh")) return 0;
}
- /* fmt=6 has no scale buffer at all (scales live in-block, scale_count 0), and
- * the fallback below would otherwise copy O floats out of a NULL host pointer. */
+ /* fmt=6 stores scales in-block; fmt=7 stores byte exponents separately. */
return !tensor->fmt || tensor->fmt==6 || cuda_ok(cudaMemcpy(tensor->scales,scales,
- (tensor->scale_count?tensor->scale_count:(size_t)tensor->O)*sizeof(float),
+ tensor_scale_bytes(tensor),
cudaMemcpyHostToDevice),"scale refresh");
}
@@ -2386,19 +2394,14 @@ extern "C" void coli_cuda_tensor_free(ColiCudaTensor *tensor) {
DeviceContext *ctx = find_ctx(tensor->device);
if (ctx) select_ctx(ctx);
if (tensor->tracked && ctx) {
- /* Must mirror the upload's accounting exactly -- literally the same
- * expression upload uses to charge (scale_count * sizeof(float), gated
- * on fmt=6 never having a separate scale buffer), so the two can no
- * longer drift independently. Over-subtracting here trips the >= guard
- * below, which silently leaves the tensor's bytes on the device counter
- * forever. */
+ /* Charge and release the same format-specific scale storage. */
size_t storage_bytes =
#ifdef COLI_ANS
tensor->compressed ? tensor->archive_bytes :
#endif
tensor->weight_bytes;
size_t bytes = storage_bytes +
- ((tensor->fmt && tensor->fmt != 6) ? tensor->scale_count * sizeof(float) : 0);
+ tensor_scale_bytes(tensor);
if (ctx->tensor_count) ctx->tensor_count--;
if (ctx->tensor_bytes >= bytes) ctx->tensor_bytes -= bytes;
}
@@ -2410,19 +2413,14 @@ extern "C" void coli_cuda_tensor_free(ColiCudaTensor *tensor) {
extern "C" size_t coli_cuda_tensor_bytes(const ColiCudaTensor *tensor) {
if (!tensor) return 0;
- /* Must mirror upload's and free's accounting exactly -- literally the same
- * expression they use (scale_count * sizeof(float), gated on fmt=6 never
- * having a separate scale buffer) -- so all three can no longer drift
- * independently. The prior `O * ng` shape over-reported for fmt=8 (real
- * footprint is (O+127)/128 * ng block scales, not O * ng) and for fmt=6
- * (which has no separate scale buffer at all). */
+ /* Logical size uses the same scale layout as upload and free. */
size_t storage_bytes =
#ifdef COLI_ANS
tensor->compressed ? tensor->archive_bytes :
#endif
tensor->weight_bytes;
return storage_bytes +
- ((tensor->fmt && tensor->fmt != 6) ? tensor->scale_count * sizeof(float) : 0);
+ tensor_scale_bytes(tensor);
}
/* What a cudaMalloc of `bytes` actually takes off the card.
@@ -2536,7 +2534,7 @@ extern "C" size_t coli_cuda_tensor_vram(const ColiCudaTensor *tensor) {
tensor->weight_bytes;
size_t total = coli_cuda_alloc_footprint(storage_bytes);
if (tensor->fmt && tensor->fmt != 6)
- total += coli_cuda_alloc_footprint(tensor->scale_count * sizeof(float));
+ total += coli_cuda_alloc_footprint(tensor_scale_bytes(tensor));
return total;
}
diff --git a/c/tests/test_mxfp4_cuda.cu b/c/tests/test_mxfp4_cuda.cu
index 6604560d..b329ea40 100644
--- a/c/tests/test_mxfp4_cuda.cu
+++ b/c/tests/test_mxfp4_cuda.cu
@@ -204,6 +204,41 @@ static void all_codes(void) {
else printf(" ok all 16 e2m1 codes decode exactly (cpu == gpu == spec)\n");
}
+/* Resident upload/update must use O*ceil(I/32) BYTES, including tails.
+ * Positive inputs avoid cancellation so every missing group affects the result. */
+static void resident_case(int I, int O) {
+ const int S = 2, rb = (I + 1) / 2, ng = (I + 31) / 32;
+ uint8_t *q = (uint8_t *)malloc((size_t)O * rb);
+ uint8_t *sc = (uint8_t *)malloc((size_t)O * ng);
+ float *x = (float *)malloc((size_t)S * I * sizeof(float));
+ float *want = (float *)malloc((size_t)S * O * sizeof(float));
+ float *got = (float *)malloc((size_t)S * O * sizeof(float));
+ memset(q, 0x22, (size_t)O * rb);
+ for (int i = 0; i < S * I; i++) x[i] = 0.25f * (1 + i % 3);
+ size_t count0, bytes0, count1, bytes1;
+ coli_cuda_stats(0, &count0, &bytes0);
+ ColiCudaTensor *t = nullptr;
+ for (int pass = 0; pass < 2; pass++) {
+ for (int i = 0; i < O * ng; i++) sc[i] = (uint8_t)(125 + (i + pass) % 5);
+ const float *scales = reinterpret_cast<const float *>(sc);
+ int ok = pass ? coli_cuda_tensor_update(t, q, scales)
+ : coli_cuda_tensor_upload(&t, q, scales, 7, I, O, 0);
+ if (!ok) { printf(" FAIL resident upload/update\n"); fails++; break; }
+ size_t expected = (size_t)O * (rb + ng);
+ coli_cuda_stats(0, &count1, &bytes1);
+ if (coli_cuda_tensor_bytes(t) != expected || count1 != count0 + 1 || bytes1 != bytes0 + expected) {
+ printf(" FAIL resident byte accounting I=%d O=%d\n", I, O); fails++;
+ }
+ mxfp4_ref(want, x, q, sc, S, I, O);
+ if (!coli_cuda_matmul(&t, got, x, nullptr, nullptr, 7, S, I, O, 0, 0)) fails++;
+ else compare_case(pass ? "resident refresh" : "resident upload", want, got, S, I, O);
+ }
+ coli_cuda_tensor_free(t);
+ coli_cuda_stats(0, &count1, &bytes1);
+ if (count0 != count1 || bytes0 != bytes1) { printf(" FAIL resident free accounting\n"); fails++; }
+ free(q); free(sc); free(x); free(want); free(got);
+}
+
int main(void) {
int ndev = 0;
if (cudaGetDeviceCount(&ndev) != cudaSuccess || ndev < 1) {
@@ -216,6 +251,8 @@ int main(void) {
return 0;
}
+ resident_case(33, 3); /* 6 exponent bytes, not 12 */
+ resident_case(257, 5); /* 45 exponent bytes, not 20 */
all_codes();
one_case("decode + matmul", 1, 64, 32, -1);
one_case("multi-row batch", 4, 128, 64, -1);