Skip to content

Run frontier MoE models on hardware you already own — pure C, zero deps, experts streamed from disk. Tiny engine, immense model. 🐦

C

fix(cuda): set weights_owned before the H2D copy so failed uploads free their buffer

monotophic7612433976bd
diff --git a/c/Makefile b/c/Makefile
index 1eeb254a..dc9652e5 100644
--- a/c/Makefile
+++ b/c/Makefile
@@ -840,7 +840,7 @@ rans: $(RANSLIB)
$(RANSLIB): tools/rans_ctypes.c rans.h
$(CC) $(CFLAGS) -fPIC -shared $< -o $@ $(LDFLAGS)
-cuda-test: backend_cuda.cu backend_cuda.h backend_gpu_compat.h tests/test_backend_cuda.cu tests/test_ragged_attention.cu tests/test_absorb_determinism.cu tests/test_mxfp4_cuda.cu tests/mxfp4_ref.c tests/test_fp8_warp_cuda.cu tests/test_fp8_cuda.cu
+cuda-test: backend_cuda.cu backend_cuda.h backend_gpu_compat.h tests/test_backend_cuda.cu tests/test_ragged_attention.cu tests/test_absorb_determinism.cu tests/test_mxfp4_cuda.cu tests/mxfp4_ref.c tests/test_fp8_warp_cuda.cu tests/test_fp8_cuda.cu tests/test_weights_owned_cuda.cu
@command -v "$(GPUCC)" >/dev/null 2>&1 || { echo "$(GPUCC_NAME) not found" >&2; exit 1; }
"$(GPUCC)" $(GPUFLAGS) backend_cuda.cu tests/test_backend_cuda.cu -o backend_cuda_test$(EXE) $(ANS_NVCC_LIBS)
./backend_cuda_test$(EXE)
@@ -870,6 +870,12 @@ cuda-test: backend_cuda.cu backend_cuda.h backend_gpu_compat.h tests/test_backen
# test file is compiled. Also runs BEFORE the MXFP4 test (same reason).
"$(GPUCC)" $(GPUFLAGS) tests/test_fp8_cuda.cu -o fp8_cuda_test$(EXE) $(ANS_NVCC_LIBS)
./fp8_cuda_test$(EXE)
+ # Upload-failure ownership: a failed H2D weight copy must release the
+ # bytes it allocated (weights_owned set by the malloc, not the copy).
+ # Includes backend_cuda.cu itself to plant the memcpy failure, so only
+ # the test file is compiled.
+ "$(GPUCC)" $(GPUFLAGS) tests/test_weights_owned_cuda.cu -o weights_owned_test$(EXE) $(ANS_NVCC_LIBS)
+ ./weights_owned_test$(EXE)
# MXFP4 (fmt=7) against the CPU decoder in quant.h. The reference is built
# as C on purpose: quant.h uses _Thread_local, which nvcc's C++ front end
# rejects, and re-implementing it for the test would defeat the comparison.
@@ -906,6 +912,7 @@ gpu-compile: backend_cuda.o
"$(GPUCC)" $(GPUFLAGS) backend_cuda.cu tests/test_ragged_attention.cu -o ragged_attention_test$(EXE)
"$(GPUCC)" $(GPUFLAGS) backend_cuda.cu tests/test_absorb_determinism.cu -o absorb_determinism_test$(EXE)
"$(GPUCC)" $(GPUFLAGS) tests/test_fp8_warp_cuda.cu -o fp8_warp_test$(EXE) $(ANS_NVCC_LIBS)
+ "$(GPUCC)" $(GPUFLAGS) tests/test_weights_owned_cuda.cu -o weights_owned_test$(EXE) $(ANS_NVCC_LIBS)
cuda-bench: backend_cuda.cu backend_cuda.h backend_gpu_compat.h tests/bench_tensor_core.cu
@command -v "$(GPUCC)" >/dev/null 2>&1 || { echo "$(GPUCC_NAME) not found" >&2; exit 1; }
diff --git a/c/backend_cuda.cu b/c/backend_cuda.cu
index 017c9db4..8f51b2f9 100644
--- a/c/backend_cuda.cu
+++ b/c/backend_cuda.cu
@@ -1367,12 +1367,18 @@ extern "C" int coli_cuda_tensor_upload(ColiCudaTensor **tensor,
t->ng = (I + 127) / 128;
t->scale_count = (size_t)((O + 127) / 128) * (size_t)t->ng;
}
- if (!cuda_ok(cudaMalloc(&t->weights, t->weight_bytes), "tensor allocation") ||
- !cuda_ok(cudaMemcpy(t->weights, weights, t->weight_bytes, cudaMemcpyHostToDevice), "tensor upload")) {
+ if (!cuda_ok(cudaMalloc(&t->weights, t->weight_bytes), "tensor allocation")) {
coli_cuda_tensor_free(t);
return 0;
}
+ /* Ownership is a fact of the allocation, not the copy: set it BEFORE the
+ * memcpy, or a failed H2D upload frees the tensor while weights_owned is
+ * still 0 and free()'s ownership gate leaks the device buffer. */
t->weights_owned=1;
+ if (!cuda_ok(cudaMemcpy(t->weights, weights, t->weight_bytes, cudaMemcpyHostToDevice), "tensor upload")) {
+ coli_cuda_tensor_free(t);
+ return 0;
+ }
if(fmt==2||fmt==4){ /* same nibble layout: offset-binary -> signed in place */
offset_to_signed_s4<<<(unsigned)((t->weight_bytes+255)/256),256>>>((uint8_t*)t->weights,t->weight_bytes);
if(!cuda_ok(cudaGetLastError(),"int4 weight conversion")){coli_cuda_tensor_free(t);return 0;}}
diff --git a/c/tests/test_weights_owned_cuda.cu b/c/tests/test_weights_owned_cuda.cu
new file mode 100644
index 00000000..68139132
--- /dev/null
+++ b/c/tests/test_weights_owned_cuda.cu
@@ -0,0 +1,189 @@
+/* weights_owned ownership-vs-copy VRAM leak (upload failure path).
+ *
+ * At the defect (U6 audit F3): coli_cuda_tensor_upload sets
+ * t->weights_owned = 1 only AFTER the H2D weight memcpy succeeds. When the
+ * cudaMalloc succeeds and the memcpy fails, the failure branch calls
+ * coli_cuda_tensor_free(t) while weights_owned is still 0, free's
+ * `weights && weights_owned` gate skips the cudaFree, and the device buffer
+ * leaks — invisibly to the ledger (tracked=0), visibly to the consumer
+ * (cudaMemGetInfo free memory drops by weight_bytes per failed upload).
+ * Ownership is a fact of the allocation, not the copy.
+ *
+ * Contract pinned here:
+ * 1. Control: N successful upload+free cycles leave device free memory
+ * unchanged (within noise), and the ledger returns to zero.
+ * 2. N upload attempts whose weight H2D memcpy fails (injected below)
+ * leave device free memory unchanged — a failed upload releases every
+ * byte it allocated. On the defect this loses weight_bytes per cycle.
+ * 3. The ledger never sees the failed uploads (charge only on success).
+ * 4. The upload path still works after the injection is disarmed.
+ *
+ * Injection seam: the backend is included directly (kernel-level pattern of
+ * tests/test_fp8_warp_cuda.cu), with every `cudaMemcpy` token in the included
+ * backend source renamed to a hook via an object-like macro. The runtime
+ * header itself is untouched: the compiler pre-includes its runtime header
+ * before this file, so the real declaration is already in scope and the
+ * macro only rewrites the backend's call sites. The hook fails HostToDevice
+ * copies of exactly the armed byte count and passes everything else through
+ * to the real runtime call, so ONLY the weight upload of the target tensor
+ * shape is hit. On HIP the compat header maps cudaMemcpy -> hipMemcpy, so
+ * the rename is applied to hipMemcpy and chains identically.
+ *
+ * Build: nvcc -O2 -std=c++17 -arch=native tests/test_weights_owned_cuda.cu -o tests/weights_owned_test
+ */
+#include <cstdio>
+#include <cstdlib>
+#include <cstring>
+#include <vector>
+
+/* Forward declaration first (the compiler's pre-included runtime header has
+ * already supplied the types), then the rename, then the backend. */
+#if defined(__HIP_PLATFORM_AMD__) || defined(__HIP__)
+extern "C" hipError_t coli_test_memcpy_hook(void *dst, const void *src,
+ size_t count, hipMemcpyKind kind);
+#define hipMemcpy coli_test_memcpy_hook
+#else
+extern "C" cudaError_t coli_test_memcpy_hook(void *dst, const void *src,
+ size_t count, enum cudaMemcpyKind kind);
+#define cudaMemcpy coli_test_memcpy_hook
+#endif
+
+/* No direct <cuda_runtime.h>: the backend include pulls backend_gpu_compat.h,
+ * which supplies the runtime surface for BOTH vendors. */
+#include "../backend_cuda.cu"
+
+#if defined(__HIP_PLATFORM_AMD__) || defined(__HIP__)
+#undef hipMemcpy
+#define COLI_REAL_MEMCPY hipMemcpy
+#else
+#undef cudaMemcpy
+#define COLI_REAL_MEMCPY cudaMemcpy
+#endif
+
+static size_t g_fail_h2d_bytes = 0; /* 0 = pass everything through */
+static int g_hook_hits = 0;
+
+extern "C" cudaError_t coli_test_memcpy_hook(void *dst, const void *src,
+ size_t count, cudaMemcpyKind kind) {
+ if (g_fail_h2d_bytes && kind == cudaMemcpyHostToDevice && count == g_fail_h2d_bytes) {
+ g_hook_hits++;
+ return cudaErrorInvalidValue;
+ }
+ return COLI_REAL_MEMCPY(dst, src, count, kind);
+}
+
+static int device_free_bytes(size_t *out) {
+ size_t free_b = 0, total_b = 0;
+ if (cudaMemGetInfo(&free_b, &total_b) != cudaSuccess) return 0;
+ *out = free_b;
+ return 1;
+}
+
+int main(int argc, char **argv) {
+ int devices[COLI_CUDA_MAX_DEVICES], ndev = argc > 1 ? argc - 1 : 1;
+ if (ndev > COLI_CUDA_MAX_DEVICES) return 2;
+ for (int i = 0; i < ndev; i++) devices[i] = argc > 1 ? std::atoi(argv[i + 1]) : 0;
+ if (!coli_cuda_init(devices, ndev)) return 77;
+ int d0 = devices[0];
+
+ /* fmt=0 (float weights, no scale buffer): weight_bytes is exactly
+ * I*O*4, giving the hook an unambiguous byte-count target, and a 64 MB
+ * signal that dwarfs allocator/driver jitter (SLACK below). */
+ const int I = 4096, O = 4096, N = 8;
+ const size_t wb = (size_t)I * O * sizeof(float); /* 64 MiB */
+ const size_t SLACK = wb / 2; /* driver noise allowance */
+ std::vector<float> w((size_t)I * O, 0.5f);
+
+ size_t count = 99, bytes = 99;
+ coli_cuda_stats(-1, &count, &bytes);
+ if (count || bytes) return 1;
+
+ /* Warmup: the first real allocation can trigger lazy driver/module
+ * setup whose memory never returns; absorb it before any sampling. */
+ {
+ ColiCudaTensor *t = nullptr;
+ if (!coli_cuda_tensor_upload(&t, w.data(), nullptr, 0, I, O, d0) || !t) {
+ std::fprintf(stderr, "warmup upload failed\n");
+ return 1;
+ }
+ coli_cuda_tensor_free(t);
+ }
+
+ /* ---- control: success path allocates and frees to zero delta -------- */
+ size_t free_before = 0, free_now = 0;
+ if (!device_free_bytes(&free_before)) return 2;
+ for (int n = 0; n < N; n++) {
+ ColiCudaTensor *t = nullptr;
+ if (!coli_cuda_tensor_upload(&t, w.data(), nullptr, 0, I, O, d0) || !t) {
+ std::fprintf(stderr, "control upload %d failed\n", n);
+ return 1;
+ }
+ coli_cuda_tensor_free(t);
+ }
+ if (!device_free_bytes(&free_now)) return 2;
+ long long control_delta = (long long)free_before - (long long)free_now;
+ std::printf("control: %d upload+free cycles of %zu MiB, free-mem delta %lld bytes\n",
+ N, wb >> 20, control_delta);
+ if (control_delta > (long long)SLACK || -control_delta > (long long)SLACK) {
+ std::fprintf(stderr, "control cycles moved device free memory by %lld bytes\n",
+ control_delta);
+ return 1;
+ }
+ coli_cuda_stats(-1, &count, &bytes);
+ if (count || bytes) {
+ std::fprintf(stderr, "ledger not restored after control cycles: %zu tensors %zu bytes\n",
+ count, bytes);
+ return 1;
+ }
+
+ /* ---- injected H2D failure: a failed upload must release its bytes --- */
+ std::fprintf(stderr, "expecting %d upload-failure lines next (deliberate injected H2D failures):\n", N);
+ g_fail_h2d_bytes = wb;
+ if (!device_free_bytes(&free_before)) return 2;
+ for (int n = 0; n < N; n++) {
+ ColiCudaTensor *t = nullptr;
+ if (coli_cuda_tensor_upload(&t, w.data(), nullptr, 0, I, O, d0) || t) {
+ std::fprintf(stderr, "upload %d ignored the injected memcpy failure\n", n);
+ g_fail_h2d_bytes = 0;
+ return 1;
+ }
+ if (!device_free_bytes(&free_now)) { g_fail_h2d_bytes = 0; return 2; }
+ std::printf("fail-cycle %d: free %zu bytes (cumulative delta %lld)\n",
+ n, free_now, (long long)free_before - (long long)free_now);
+ }
+ g_fail_h2d_bytes = 0;
+ if (g_hook_hits != N) {
+ std::fprintf(stderr, "injection seam engaged %d times, want %d\n", g_hook_hits, N);
+ return 1;
+ }
+ long long leaked = (long long)free_before - (long long)free_now;
+ coli_cuda_stats(-1, &count, &bytes);
+ if (count || bytes) {
+ std::fprintf(stderr, "failed uploads reached the ledger: %zu tensors %zu bytes\n",
+ count, bytes);
+ return 1;
+ }
+ if (leaked > (long long)SLACK) {
+ std::fprintf(stderr,
+ "FAIL: %d failed uploads leaked %lld bytes of device memory "
+ "(%.1f MiB, ~%zu MiB per cycle expected on the defect)\n",
+ N, leaked, leaked / (1024.0 * 1024.0), wb >> 20);
+ return 1;
+ }
+
+ /* ---- disarmed: the upload path still works ---------------------------- */
+ {
+ ColiCudaTensor *t = nullptr;
+ if (!coli_cuda_tensor_upload(&t, w.data(), nullptr, 0, I, O, d0) || !t) {
+ std::fprintf(stderr, "healthy upload after disarm failed\n");
+ return 1;
+ }
+ coli_cuda_tensor_free(t);
+ coli_cuda_stats(-1, &count, &bytes);
+ if (count || bytes) return 1;
+ }
+
+ std::printf("weights_owned upload-failure test passed (%d fail cycles, %lld bytes cumulative)\n",
+ N, leaked);
+ return 0;
+}