Skip to content

Run frontier MoE models on hardware you already own — pure C, zero deps, experts streamed from disk. Tiny engine, immense model. 🐦

C

fix(cuda): unwind partially initialized device contexts

ZacharyZcR33900c25b8e1
diff --git a/c/Makefile b/c/Makefile
index 341ad971..7f3d044c 100644
--- a/c/Makefile
+++ b/c/Makefile
@@ -956,10 +956,12 @@ rans: $(RANSLIB)
$(RANSLIB): tools/rans_ctypes.c rans.h
$(CC) $(CFLAGS) -fPIC -shared $< -o $@ $(LDFLAGS)
-cuda-test: backend_cuda.cu backend_cuda.h backend_gpu_compat.h tests/test_backend_cuda.cu tests/test_ragged_attention.cu tests/test_absorb_determinism.cu tests/test_mxfp4_cuda.cu tests/mxfp4_ref.c tests/test_fp8_warp_cuda.cu tests/test_fp8_cuda.cu tests/test_weights_owned_cuda.cu tests/test_cuda_fmt_trap_cuda.cu tests/test_alloc_footprint_cuda.cu
+cuda-test: backend_cuda.cu backend_cuda.h backend_gpu_compat.h tests/test_backend_cuda.cu tests/test_ragged_attention.cu tests/test_absorb_determinism.cu tests/test_mxfp4_cuda.cu tests/mxfp4_ref.c tests/test_fp8_warp_cuda.cu tests/test_fp8_cuda.cu tests/test_weights_owned_cuda.cu tests/test_cuda_fmt_trap_cuda.cu tests/test_alloc_footprint_cuda.cu tests/test_cuda_init_failure.cu
@command -v "$(GPUCC)" >/dev/null 2>&1 || { echo "$(GPUCC_NAME) not found" >&2; exit 1; }
"$(GPUCC)" $(GPUFLAGS) backend_cuda.cu tests/test_backend_cuda.cu -o backend_cuda_test$(EXE) $(ANS_NVCC_LIBS) $(GPU_TEST_LIBS)
./backend_cuda_test$(EXE)
+ "$(GPUCC)" $(GPUFLAGS) tests/test_cuda_init_failure.cu -o cuda_init_failure_test$(EXE) $(ANS_NVCC_LIBS) $(GPU_TEST_LIBS)
+ ./cuda_init_failure_test$(EXE)
"$(GPUCC)" $(GPUFLAGS) backend_cuda.cu tests/test_ragged_attention.cu -o ragged_attention_test$(EXE) $(GPU_TEST_LIBS)
./ragged_attention_test$(EXE)
# weight_at's device-side refusal (the __trap()) on real silicon. The
diff --git a/c/backend_cuda.cu b/c/backend_cuda.cu
index 07d5dfc4..f2fa210a 100644
--- a/c/backend_cuda.cu
+++ b/c/backend_cuda.cu
@@ -1242,28 +1242,29 @@ extern "C" int coli_cuda_init(const int *devices, int count) {
int available = 0;
if (!devices || count < 1 || count > COLI_CUDA_MAX_DEVICES) return 0;
if (!cuda_ok(cudaGetDeviceCount(&available), "device discovery")) return 0;
- g_nctx = 0;
+ /* Validate the whole list before creating resources or replacing state. */
for (int i = 0; i < count; i++) {
- int device = devices[i];
- if (device < 0 || device >= available) {
- std::fprintf(stderr, "[CUDA] invalid device %d (available: 0..%d)\n", device, available - 1);
- g_nctx = 0;
+ if (devices[i] < 0 || devices[i] >= available) {
+ std::fprintf(stderr, "[CUDA] invalid device %d (available: 0..%d)\n", devices[i], available - 1);
return 0;
}
- if (find_ctx(device)) {
- std::fprintf(stderr, "[CUDA] duplicate device %d\n", device);
- g_nctx = 0;
+ for (int j = 0; j < i; j++) if (devices[j] == devices[i]) {
+ std::fprintf(stderr, "[CUDA] duplicate device %d\n", devices[i]);
return 0;
}
+ }
+ g_nctx = 0;
+ for (int i = 0; i < count; i++) {
+ int device = devices[i];
DeviceContext *ctx = &g_ctx[g_nctx];
*ctx = {};
ctx->device = device;
- if (!select_ctx(ctx)) { g_nctx = 0; return 0; }
+ if (!select_ctx(ctx)) { coli_cuda_shutdown(); return 0; }
cudaDeviceProp prop{};
- if (!cuda_ok(cudaGetDeviceProperties(&prop, device), "device properties")) { g_nctx = 0; return 0; }
+ if (!cuda_ok(cudaGetDeviceProperties(&prop, device), "device properties")) { coli_cuda_shutdown(); return 0; }
ctx->compute_major=prop.major;ctx->compute_minor=prop.minor;
if(!cuda_ok(cudaStreamCreateWithFlags(&ctx->stream,cudaStreamNonBlocking),"stream creation")){
- g_nctx=0;return 0;
+ coli_cuda_shutdown();return 0;
}
#ifdef COLI_ANS
if(std::getenv("CUDA_RAW_EXPERTS")){
diff --git a/c/backend_gpu_compat.h b/c/backend_gpu_compat.h
index 04b64dc6..f3a3508f 100644
--- a/c/backend_gpu_compat.h
+++ b/c/backend_gpu_compat.h
@@ -60,6 +60,7 @@ namespace nvcuda { namespace wmma = ::rocwmma; }
#endif
#define cudaError_t hipError_t
#define cudaSuccess hipSuccess
+#define cudaErrorMemoryAllocation hipErrorOutOfMemory
#define cudaGetErrorString hipGetErrorString
#define cudaGetLastError hipGetLastError
#define cudaSetDevice hipSetDevice
diff --git a/c/tests/test_cuda_init_failure.cu b/c/tests/test_cuda_init_failure.cu
new file mode 100644
index 00000000..93e97be6
--- /dev/null
+++ b/c/tests/test_cuda_init_failure.cu
@@ -0,0 +1,47 @@
+/* Two logical devices mapped to one real GPU, with real stream lifetimes.
+ * This tests initialization rollback, not physical multi-GPU execution. */
+#include "../backend_gpu_compat.h"
+#include <cstdio>
+static int attempts, live_streams, fail_stream;
+static cudaError_t real_count(int *n) { return cudaGetDeviceCount(n); }
+static cudaError_t test_count(int *n) { *n=2;return cudaSuccess; }
+static cudaError_t test_select(int) { return cudaSetDevice(0); }
+static cudaError_t test_props(cudaDeviceProp *p,int) { return cudaGetDeviceProperties(p,0); }
+static cudaError_t test_create(cudaStream_t *s,unsigned flags) {
+ if(++attempts==fail_stream) return cudaErrorMemoryAllocation;
+ cudaError_t e=cudaStreamCreateWithFlags(s,flags);if(e==cudaSuccess)live_streams++;return e;
+}
+static cudaError_t test_destroy(cudaStream_t s) {
+ cudaError_t e=cudaStreamDestroy(s);if(e==cudaSuccess)live_streams--;return e;
+}
+#undef cudaGetDeviceCount
+#undef cudaSetDevice
+#undef cudaGetDeviceProperties
+#undef cudaStreamCreateWithFlags
+#undef cudaStreamDestroy
+#define cudaGetDeviceCount test_count
+#define cudaSetDevice test_select
+#define cudaGetDeviceProperties test_props
+#define cudaStreamCreateWithFlags test_create
+#define cudaStreamDestroy test_destroy
+#include "../backend_cuda.cu"
+int main(void) {
+ int count=0;if(real_count(&count)!=cudaSuccess||!count){puts("SKIP: no GPU");return 0;}
+ int ids[]={0,1}, duplicate[]={0,0},invalid[]={0,2};
+ for(int round=0;round<2;round++){
+ attempts=0;
+ if(coli_cuda_init(duplicate,2)||attempts||live_streams||coli_cuda_device_count()) return 1;
+ if(coli_cuda_init(invalid,2)||attempts||live_streams||coli_cuda_device_count()) return 2;
+ for(fail_stream=1;fail_stream<=2;fail_stream++){
+ attempts=0;
+ if(coli_cuda_init(ids,2)||live_streams||coli_cuda_device_count()) return 3;
+ }
+ fail_stream=0;attempts=0;
+ if(!coli_cuda_init(ids,2)||live_streams!=2||coli_cuda_device_count()!=2) return 4;
+ /* Rejected input must leave an existing backend reachable. */
+ if(coli_cuda_init(invalid,2)||live_streams!=2||coli_cuda_device_count()!=2) return 5;
+ if(coli_cuda_init(duplicate,2)||live_streams!=2||coli_cuda_device_count()!=2) return 6;
+ coli_cuda_shutdown();if(live_streams||coli_cuda_device_count()) return 7;
+ }
+ puts("CUDA initialization rollback: PASS");return 0;
+}