Run frontier MoE models on hardware you already own — pure C, zero deps, experts streamed from disk. Tiny engine, immense model. 🐦
C
fix(cuda): set weights_owned before the H2D copy so failed uploads free their buffer
monotophic7612433976bd
diff --git a/c/Makefile b/c/Makefileindex 1eeb254a..dc9652e5 100644--- a/c/Makefile+++ b/c/Makefile@@ -840,7 +840,7 @@ rans: $(RANSLIB)$(RANSLIB): tools/rans_ctypes.c rans.h$(CC) $(CFLAGS) -fPIC -shared $< -o $@ $(LDFLAGS)-cuda-test: backend_cuda.cu backend_cuda.h backend_gpu_compat.h tests/test_backend_cuda.cu tests/test_ragged_attention.cu tests/test_absorb_determinism.cu tests/test_mxfp4_cuda.cu tests/mxfp4_ref.c tests/test_fp8_warp_cuda.cu tests/test_fp8_cuda.cu+cuda-test: backend_cuda.cu backend_cuda.h backend_gpu_compat.h tests/test_backend_cuda.cu tests/test_ragged_attention.cu tests/test_absorb_determinism.cu tests/test_mxfp4_cuda.cu tests/mxfp4_ref.c tests/test_fp8_warp_cuda.cu tests/test_fp8_cuda.cu tests/test_weights_owned_cuda.cu@command -v "$(GPUCC)" >/dev/null 2>&1 || { echo "$(GPUCC_NAME) not found" >&2; exit 1; }"$(GPUCC)" $(GPUFLAGS) backend_cuda.cu tests/test_backend_cuda.cu -o backend_cuda_test$(EXE) $(ANS_NVCC_LIBS)./backend_cuda_test$(EXE)@@ -870,6 +870,12 @@ cuda-test: backend_cuda.cu backend_cuda.h backend_gpu_compat.h tests/test_backen# test file is compiled. Also runs BEFORE the MXFP4 test (same reason)."$(GPUCC)" $(GPUFLAGS) tests/test_fp8_cuda.cu -o fp8_cuda_test$(EXE) $(ANS_NVCC_LIBS)./fp8_cuda_test$(EXE)+ # Upload-failure ownership: a failed H2D weight copy must release the+ # bytes it allocated (weights_owned set by the malloc, not the copy).+ # Includes backend_cuda.cu itself to plant the memcpy failure, so only+ # the test file is compiled.+ "$(GPUCC)" $(GPUFLAGS) tests/test_weights_owned_cuda.cu -o weights_owned_test$(EXE) $(ANS_NVCC_LIBS)+ ./weights_owned_test$(EXE)# MXFP4 (fmt=7) against the CPU decoder in quant.h. The reference is built# as C on purpose: quant.h uses _Thread_local, which nvcc's C++ front end# rejects, and re-implementing it for the test would defeat the comparison.@@ -906,6 +912,7 @@ gpu-compile: backend_cuda.o"$(GPUCC)" $(GPUFLAGS) backend_cuda.cu tests/test_ragged_attention.cu -o ragged_attention_test$(EXE)"$(GPUCC)" $(GPUFLAGS) backend_cuda.cu tests/test_absorb_determinism.cu -o absorb_determinism_test$(EXE)"$(GPUCC)" $(GPUFLAGS) tests/test_fp8_warp_cuda.cu -o fp8_warp_test$(EXE) $(ANS_NVCC_LIBS)+ "$(GPUCC)" $(GPUFLAGS) tests/test_weights_owned_cuda.cu -o weights_owned_test$(EXE) $(ANS_NVCC_LIBS)cuda-bench: backend_cuda.cu backend_cuda.h backend_gpu_compat.h tests/bench_tensor_core.cu@command -v "$(GPUCC)" >/dev/null 2>&1 || { echo "$(GPUCC_NAME) not found" >&2; exit 1; }diff --git a/c/backend_cuda.cu b/c/backend_cuda.cuindex 017c9db4..8f51b2f9 100644--- a/c/backend_cuda.cu+++ b/c/backend_cuda.cu@@ -1367,12 +1367,18 @@ extern "C" int coli_cuda_tensor_upload(ColiCudaTensor **tensor,t->ng = (I + 127) / 128;t->scale_count = (size_t)((O + 127) / 128) * (size_t)t->ng;}- if (!cuda_ok(cudaMalloc(&t->weights, t->weight_bytes), "tensor allocation") ||- !cuda_ok(cudaMemcpy(t->weights, weights, t->weight_bytes, cudaMemcpyHostToDevice), "tensor upload")) {+ if (!cuda_ok(cudaMalloc(&t->weights, t->weight_bytes), "tensor allocation")) {coli_cuda_tensor_free(t);return 0;}+ /* Ownership is a fact of the allocation, not the copy: set it BEFORE the+ * memcpy, or a failed H2D upload frees the tensor while weights_owned is+ * still 0 and free()'s ownership gate leaks the device buffer. */t->weights_owned=1;+ if (!cuda_ok(cudaMemcpy(t->weights, weights, t->weight_bytes, cudaMemcpyHostToDevice), "tensor upload")) {+ coli_cuda_tensor_free(t);+ return 0;+ }if(fmt==2||fmt==4){ /* same nibble layout: offset-binary -> signed in place */offset_to_signed_s4<<<(unsigned)((t->weight_bytes+255)/256),256>>>((uint8_t*)t->weights,t->weight_bytes);if(!cuda_ok(cudaGetLastError(),"int4 weight conversion")){coli_cuda_tensor_free(t);return 0;}}diff --git a/c/tests/test_weights_owned_cuda.cu b/c/tests/test_weights_owned_cuda.cunew file mode 100644index 00000000..68139132--- /dev/null+++ b/c/tests/test_weights_owned_cuda.cu@@ -0,0 +1,189 @@+/* weights_owned ownership-vs-copy VRAM leak (upload failure path).+ *+ * At the defect (U6 audit F3): coli_cuda_tensor_upload sets+ * t->weights_owned = 1 only AFTER the H2D weight memcpy succeeds. When the+ * cudaMalloc succeeds and the memcpy fails, the failure branch calls+ * coli_cuda_tensor_free(t) while weights_owned is still 0, free's+ * `weights && weights_owned` gate skips the cudaFree, and the device buffer+ * leaks — invisibly to the ledger (tracked=0), visibly to the consumer+ * (cudaMemGetInfo free memory drops by weight_bytes per failed upload).+ * Ownership is a fact of the allocation, not the copy.+ *+ * Contract pinned here:+ * 1. Control: N successful upload+free cycles leave device free memory+ * unchanged (within noise), and the ledger returns to zero.+ * 2. N upload attempts whose weight H2D memcpy fails (injected below)+ * leave device free memory unchanged — a failed upload releases every+ * byte it allocated. On the defect this loses weight_bytes per cycle.+ * 3. The ledger never sees the failed uploads (charge only on success).+ * 4. The upload path still works after the injection is disarmed.+ *+ * Injection seam: the backend is included directly (kernel-level pattern of+ * tests/test_fp8_warp_cuda.cu), with every `cudaMemcpy` token in the included+ * backend source renamed to a hook via an object-like macro. The runtime+ * header itself is untouched: the compiler pre-includes its runtime header+ * before this file, so the real declaration is already in scope and the+ * macro only rewrites the backend's call sites. The hook fails HostToDevice+ * copies of exactly the armed byte count and passes everything else through+ * to the real runtime call, so ONLY the weight upload of the target tensor+ * shape is hit. On HIP the compat header maps cudaMemcpy -> hipMemcpy, so+ * the rename is applied to hipMemcpy and chains identically.+ *+ * Build: nvcc -O2 -std=c++17 -arch=native tests/test_weights_owned_cuda.cu -o tests/weights_owned_test+ */+#include <cstdio>+#include <cstdlib>+#include <cstring>+#include <vector>++/* Forward declaration first (the compiler's pre-included runtime header has+ * already supplied the types), then the rename, then the backend. */+#if defined(__HIP_PLATFORM_AMD__) || defined(__HIP__)+extern "C" hipError_t coli_test_memcpy_hook(void *dst, const void *src,+ size_t count, hipMemcpyKind kind);+#define hipMemcpy coli_test_memcpy_hook+#else+extern "C" cudaError_t coli_test_memcpy_hook(void *dst, const void *src,+ size_t count, enum cudaMemcpyKind kind);+#define cudaMemcpy coli_test_memcpy_hook+#endif++/* No direct <cuda_runtime.h>: the backend include pulls backend_gpu_compat.h,+ * which supplies the runtime surface for BOTH vendors. */+#include "../backend_cuda.cu"++#if defined(__HIP_PLATFORM_AMD__) || defined(__HIP__)+#undef hipMemcpy+#define COLI_REAL_MEMCPY hipMemcpy+#else+#undef cudaMemcpy+#define COLI_REAL_MEMCPY cudaMemcpy+#endif++static size_t g_fail_h2d_bytes = 0; /* 0 = pass everything through */+static int g_hook_hits = 0;++extern "C" cudaError_t coli_test_memcpy_hook(void *dst, const void *src,+ size_t count, cudaMemcpyKind kind) {+ if (g_fail_h2d_bytes && kind == cudaMemcpyHostToDevice && count == g_fail_h2d_bytes) {+ g_hook_hits++;+ return cudaErrorInvalidValue;+ }+ return COLI_REAL_MEMCPY(dst, src, count, kind);+}++static int device_free_bytes(size_t *out) {+ size_t free_b = 0, total_b = 0;+ if (cudaMemGetInfo(&free_b, &total_b) != cudaSuccess) return 0;+ *out = free_b;+ return 1;+}++int main(int argc, char **argv) {+ int devices[COLI_CUDA_MAX_DEVICES], ndev = argc > 1 ? argc - 1 : 1;+ if (ndev > COLI_CUDA_MAX_DEVICES) return 2;+ for (int i = 0; i < ndev; i++) devices[i] = argc > 1 ? std::atoi(argv[i + 1]) : 0;+ if (!coli_cuda_init(devices, ndev)) return 77;+ int d0 = devices[0];++ /* fmt=0 (float weights, no scale buffer): weight_bytes is exactly+ * I*O*4, giving the hook an unambiguous byte-count target, and a 64 MB+ * signal that dwarfs allocator/driver jitter (SLACK below). */+ const int I = 4096, O = 4096, N = 8;+ const size_t wb = (size_t)I * O * sizeof(float); /* 64 MiB */+ const size_t SLACK = wb / 2; /* driver noise allowance */+ std::vector<float> w((size_t)I * O, 0.5f);++ size_t count = 99, bytes = 99;+ coli_cuda_stats(-1, &count, &bytes);+ if (count || bytes) return 1;++ /* Warmup: the first real allocation can trigger lazy driver/module+ * setup whose memory never returns; absorb it before any sampling. */+ {+ ColiCudaTensor *t = nullptr;+ if (!coli_cuda_tensor_upload(&t, w.data(), nullptr, 0, I, O, d0) || !t) {+ std::fprintf(stderr, "warmup upload failed\n");+ return 1;+ }+ coli_cuda_tensor_free(t);+ }++ /* ---- control: success path allocates and frees to zero delta -------- */+ size_t free_before = 0, free_now = 0;+ if (!device_free_bytes(&free_before)) return 2;+ for (int n = 0; n < N; n++) {+ ColiCudaTensor *t = nullptr;+ if (!coli_cuda_tensor_upload(&t, w.data(), nullptr, 0, I, O, d0) || !t) {+ std::fprintf(stderr, "control upload %d failed\n", n);+ return 1;+ }+ coli_cuda_tensor_free(t);+ }+ if (!device_free_bytes(&free_now)) return 2;+ long long control_delta = (long long)free_before - (long long)free_now;+ std::printf("control: %d upload+free cycles of %zu MiB, free-mem delta %lld bytes\n",+ N, wb >> 20, control_delta);+ if (control_delta > (long long)SLACK || -control_delta > (long long)SLACK) {+ std::fprintf(stderr, "control cycles moved device free memory by %lld bytes\n",+ control_delta);+ return 1;+ }+ coli_cuda_stats(-1, &count, &bytes);+ if (count || bytes) {+ std::fprintf(stderr, "ledger not restored after control cycles: %zu tensors %zu bytes\n",+ count, bytes);+ return 1;+ }++ /* ---- injected H2D failure: a failed upload must release its bytes --- */+ std::fprintf(stderr, "expecting %d upload-failure lines next (deliberate injected H2D failures):\n", N);+ g_fail_h2d_bytes = wb;+ if (!device_free_bytes(&free_before)) return 2;+ for (int n = 0; n < N; n++) {+ ColiCudaTensor *t = nullptr;+ if (coli_cuda_tensor_upload(&t, w.data(), nullptr, 0, I, O, d0) || t) {+ std::fprintf(stderr, "upload %d ignored the injected memcpy failure\n", n);+ g_fail_h2d_bytes = 0;+ return 1;+ }+ if (!device_free_bytes(&free_now)) { g_fail_h2d_bytes = 0; return 2; }+ std::printf("fail-cycle %d: free %zu bytes (cumulative delta %lld)\n",+ n, free_now, (long long)free_before - (long long)free_now);+ }+ g_fail_h2d_bytes = 0;+ if (g_hook_hits != N) {+ std::fprintf(stderr, "injection seam engaged %d times, want %d\n", g_hook_hits, N);+ return 1;+ }+ long long leaked = (long long)free_before - (long long)free_now;+ coli_cuda_stats(-1, &count, &bytes);+ if (count || bytes) {+ std::fprintf(stderr, "failed uploads reached the ledger: %zu tensors %zu bytes\n",+ count, bytes);+ return 1;+ }+ if (leaked > (long long)SLACK) {+ std::fprintf(stderr,+ "FAIL: %d failed uploads leaked %lld bytes of device memory "+ "(%.1f MiB, ~%zu MiB per cycle expected on the defect)\n",+ N, leaked, leaked / (1024.0 * 1024.0), wb >> 20);+ return 1;+ }++ /* ---- disarmed: the upload path still works ---------------------------- */+ {+ ColiCudaTensor *t = nullptr;+ if (!coli_cuda_tensor_upload(&t, w.data(), nullptr, 0, I, O, d0) || !t) {+ std::fprintf(stderr, "healthy upload after disarm failed\n");+ return 1;+ }+ coli_cuda_tensor_free(t);+ coli_cuda_stats(-1, &count, &bytes);+ if (count || bytes) return 1;+ }++ std::printf("weights_owned upload-failure test passed (%d fail cycles, %lld bytes cumulative)\n",+ N, leaked);+ return 0;+}