Run frontier MoE models on hardware you already own — pure C, zero deps, experts streamed from disk. Tiny engine, immense model. 🐦
C
cuda: reuse streaming MXFP4 weight scratch
ZacharyZcRb6ac01e20ddd
diff --git a/c/backend_cuda.cu b/c/backend_cuda.cuindex 07d5dfc4..d3536336 100644--- a/c/backend_cuda.cu+++ b/c/backend_cuda.cu@@ -82,6 +82,9 @@ typedef struct {int compute_major,compute_minor;float *x, *y, *gate, *up;size_t x_cap, y_cap, gate_cap, up_cap;+ /* Streaming MXFP4 weights are refreshed on every call; only storage is reused. */+ void *mxfp4_weights, *mxfp4_scales;+ size_t mxfp4_weights_cap, mxfp4_scales_cap;/* Staging of the resident dense matvec (coli_cuda_matmul), apart from* x/y: the expert group (coli_cuda_expert_group_issue) runs on ctx->stream* asynchronously while the engine's thread keeps computing -- qwen38's@@ -1288,6 +1291,8 @@ extern "C" void coli_cuda_shutdown(void) {for (int i = 0; i < g_nctx; i++) {DeviceContext *ctx = &g_ctx[i];if (!select_ctx(ctx)) continue;+ if (ctx->mxfp4_weights) cudaFree(ctx->mxfp4_weights);+ if (ctx->mxfp4_scales) cudaFree(ctx->mxfp4_scales);if (ctx->x) cudaFree(ctx->x);if (ctx->y) cudaFree(ctx->y);if (ctx->dx) cudaFree(ctx->dx);@@ -1312,6 +1317,8 @@ extern "C" void coli_cuda_shutdown(void) {ctx->ans_scratch=nullptr;ctx->ans_chunks=nullptr;ctx->ans_raw=nullptr;ctx->ans_raw_cap=0;ctx->ans_host=nullptr;ctx->ans_host_cap=0;ctx->ans_copy_pending=0;#endif+ ctx->mxfp4_weights = ctx->mxfp4_scales = nullptr;+ ctx->mxfp4_weights_cap = ctx->mxfp4_scales_cap = 0;ctx->x = ctx->y = ctx->gate = ctx->up = nullptr;ctx->dx = ctx->dy = nullptr; ctx->dx_cap = ctx->dy_cap = 0;ctx->qx=nullptr; ctx->qscale=nullptr;@@ -1728,9 +1735,10 @@ extern "C" int coli_cuda_matmul_mxfp4(float *y, const float *x,size_t wb = (size_t)O * rb, sb = (size_t)O * ng;size_t xb = (size_t)S * I * sizeof(float), yb = (size_t)S * O * sizeof(float);- uint8_t *dw = nullptr, *ds = nullptr;- if (!cuda_ok(cudaMalloc(&dw, wb), "mxfp4 weight alloc")) return 0;- if (!cuda_ok(cudaMalloc(&ds, sb), "mxfp4 scale alloc")) { cudaFree(dw); return 0; }+ if (!reserve_bytes(&ctx->mxfp4_weights, &ctx->mxfp4_weights_cap, wb) ||+ !reserve_bytes(&ctx->mxfp4_scales, &ctx->mxfp4_scales_cap, sb)) return 0;+ uint8_t *dw = static_cast<uint8_t *>(ctx->mxfp4_weights);+ uint8_t *ds = static_cast<uint8_t *>(ctx->mxfp4_scales);int ok = reserve(&ctx->x, &ctx->x_cap, xb) && reserve(&ctx->y, &ctx->y_cap, yb) &&cuda_ok(cudaMemcpy(dw, q4, wb, cudaMemcpyHostToDevice), "mxfp4 weight upload") &&@@ -1743,8 +1751,6 @@ extern "C" int coli_cuda_matmul_mxfp4(float *y, const float *x,ok = cuda_ok(cudaGetLastError(), "mxfp4 launch") &&cuda_ok(cudaMemcpy(y, ctx->y, yb, cudaMemcpyDeviceToHost), "mxfp4 output download");}- cudaFree(dw);- cudaFree(ds);return ok;}diff --git a/c/tests/test_mxfp4_cuda.cu b/c/tests/test_mxfp4_cuda.cuindex 6604560d..a5bb3a01 100644--- a/c/tests/test_mxfp4_cuda.cu+++ b/c/tests/test_mxfp4_cuda.cu@@ -116,6 +116,13 @@ static void one_case(const char *what, int S, int I, int O, int fixed_exp) {}compare_case(what, y_cpu, y_gpu, S, I, O);+ /* The engine recycles host slots: identical addresses must upload fresh+ * bytes and scales, even when device scratch already has enough capacity. */+ memset(q4, 0x22, (size_t)O * rb);+ memset(e8, 127, (size_t)O * ng);+ mxfp4_ref(y_cpu, x, q4, e8, S, I, O);+ if (!coli_cuda_matmul_mxfp4(y_gpu, x, q4, e8, S, I, O)) fails++;+ else compare_case("recycled host slot", y_cpu, y_gpu, S, I, O);done:free(q4); free(e8); free(x); free(y_cpu); free(y_gpu);}@@ -231,6 +238,13 @@ int main(void) {mixed_254_255();one_case("exponent 127 -> unit scale", 1, 64, 16, 127);+ coli_cuda_shutdown();+ if (!coli_cuda_init(&dev0, 1)) fails++;+ else {+ one_case("after shutdown/reinit", 2, 128, 32, 127);+ coli_cuda_shutdown();+ }+printf(fails ? "test_mxfp4_cuda: %d failure(s)\n" : "test_mxfp4_cuda: ok\n", fails);return fails != 0;}