Run frontier MoE models on hardware you already own — pure C, zero deps, experts streamed from disk. Tiny engine, immense model. 🐦
C
gpu: gate grouped_s4_wmma on the s4 fragment type and sm_75, not on WMMA
JustVugg19af0fc28dcf
diff --git a/c/Makefile b/c/Makefileindex ba136a48..ae176034 100644--- a/c/Makefile+++ b/c/Makefile@@ -577,6 +577,10 @@ CUDA_OBJ = backend_cuda.oGPUCC = $(HIPCC)GPUFLAGS = $(HIPCCFLAGS)GPUCC_NAME = hipcc (set ROCM_HOME or HIPCC)+# The kernel test binaries link the HIP runtime explicitly: hipcc does not add+# it for a plain link the way nvcc adds cudart (#1499: `make hip-test` failed+# with undefined hipStreamSynchronize and the suite never ran on ROCm).+GPU_TEST_LIBS = -L$(ROCM_HOME)/lib -lamdhip64# rocWMMA needs matrix cores (MFMA gfx908+ / WMMA gfx11xx); on these archs its# headers static_assert (ROCm/TheRock#1944). Compile the WMMA paths out and# let backend_gpu_compat.h fall back via COLI_GPU_HAS_WMMA=0. The flag must@@ -897,23 +901,23 @@ $(RANSLIB): tools/rans_ctypes.c rans.hcuda-test: backend_cuda.cu backend_cuda.h backend_gpu_compat.h tests/test_backend_cuda.cu tests/test_ragged_attention.cu tests/test_absorb_determinism.cu tests/test_mxfp4_cuda.cu tests/mxfp4_ref.c tests/test_fp8_warp_cuda.cu tests/test_fp8_cuda.cu tests/test_weights_owned_cuda.cu tests/test_cuda_fmt_trap_cuda.cu tests/test_alloc_footprint_cuda.cu@command -v "$(GPUCC)" >/dev/null 2>&1 || { echo "$(GPUCC_NAME) not found" >&2; exit 1; }- "$(GPUCC)" $(GPUFLAGS) backend_cuda.cu tests/test_backend_cuda.cu -o backend_cuda_test$(EXE) $(ANS_NVCC_LIBS)+ "$(GPUCC)" $(GPUFLAGS) backend_cuda.cu tests/test_backend_cuda.cu -o backend_cuda_test$(EXE) $(ANS_NVCC_LIBS) $(GPU_TEST_LIBS)./backend_cuda_test$(EXE)- "$(GPUCC)" $(GPUFLAGS) backend_cuda.cu tests/test_ragged_attention.cu -o ragged_attention_test$(EXE)+ "$(GPUCC)" $(GPUFLAGS) backend_cuda.cu tests/test_ragged_attention.cu -o ragged_attention_test$(EXE) $(GPU_TEST_LIBS)./ragged_attention_test$(EXE)# weight_at's device-side refusal (the __trap()) on real silicon. The# host-side half is tests/test_cuda_fmt_guard.c, which runs in the CPU# suite and cannot execute device code. Includes backend_cuda.cu itself:# weight_at is file-static, and the trap is unreachable through the public# API by design. Re-execs itself per probe (the trap poisons the context).- "$(GPUCC)" $(GPUFLAGS) tests/test_cuda_fmt_trap_cuda.cu -o cuda_fmt_trap_test$(EXE) $(ANS_NVCC_LIBS)+ "$(GPUCC)" $(GPUFLAGS) tests/test_cuda_fmt_trap_cuda.cu -o cuda_fmt_trap_test$(EXE) $(ANS_NVCC_LIBS) $(GPU_TEST_LIBS)./cuda_fmt_trap_test$(EXE)# fmt=8 warp kernels (COLI_CUDA_F8_WARP): decode sweep, census + tail# shapes, S-invariance, NaN/denormal policy. Includes backend_cuda.cu# itself (kernel-level oracle), so only the test file is compiled.# Runs BEFORE the MXFP4 test: that one has a pre-existing exponent-255# failure on some hosts and would otherwise abort the recipe first.- "$(GPUCC)" $(GPUFLAGS) tests/test_fp8_warp_cuda.cu -o fp8_warp_test$(EXE) $(ANS_NVCC_LIBS)+ "$(GPUCC)" $(GPUFLAGS) tests/test_fp8_warp_cuda.cu -o fp8_warp_test$(EXE) $(ANS_NVCC_LIBS) $(GPU_TEST_LIBS)./fp8_warp_test$(EXE)# The batch and ragged absorb kernels must reproduce their own output: they# reduce in a fixed tree order with no atomics, so two identical launches@@ -925,18 +929,18 @@ cuda-test: backend_cuda.cu backend_cuda.h backend_gpu_compat.h tests/test_backen# test in the recipe, and it should not cost the broader kernel suites if# it trips. It stays BEFORE MXFP4 so it still runs on the hosts where that# one is expected to fail.- "$(GPUCC)" $(GPUFLAGS) backend_cuda.cu tests/test_absorb_determinism.cu -o absorb_determinism_test$(EXE)+ "$(GPUCC)" $(GPUFLAGS) backend_cuda.cu tests/test_absorb_determinism.cu -o absorb_determinism_test$(EXE) $(GPU_TEST_LIBS)./absorb_determinism_test$(EXE)# fmt=8 (fp8-e4m3) kernel oracle + public API (LUT gate, dense, expert# group, byte accounting). Includes backend_cuda.cu itself, so only the# test file is compiled. Also runs BEFORE the MXFP4 test (same reason).- "$(GPUCC)" $(GPUFLAGS) tests/test_fp8_cuda.cu -o fp8_cuda_test$(EXE) $(ANS_NVCC_LIBS)+ "$(GPUCC)" $(GPUFLAGS) tests/test_fp8_cuda.cu -o fp8_cuda_test$(EXE) $(ANS_NVCC_LIBS) $(GPU_TEST_LIBS)./fp8_cuda_test$(EXE)# Upload-failure ownership: a failed H2D weight copy must release the# bytes it allocated (weights_owned set by the malloc, not the copy).# Includes backend_cuda.cu itself to plant the memcpy failure, so only# the test file is compiled.- "$(GPUCC)" $(GPUFLAGS) tests/test_weights_owned_cuda.cu -o weights_owned_test$(EXE) $(ANS_NVCC_LIBS)+ "$(GPUCC)" $(GPUFLAGS) tests/test_weights_owned_cuda.cu -o weights_owned_test$(EXE) $(ANS_NVCC_LIBS) $(GPU_TEST_LIBS)./weights_owned_test$(EXE)# MXFP4 (fmt=7) against the CPU decoder in quant.h. The reference is built# as C on purpose: quant.h uses _Thread_local, which nvcc's C++ front end@@ -955,14 +959,14 @@ cuda-test: backend_cuda.cu backend_cuda.h backend_gpu_compat.h tests/test_backen# below no longer pulls the runtime, and CI could not catch it because CI# never runs cuda-test. -Xclang appears nowhere else in CFLAGS.$(CC) $(filter-out -Xclang -fopenmp,$(CFLAGS)) -fno-openmp -c tests/mxfp4_ref.c -o tests/mxfp4_ref.o- "$(GPUCC)" $(GPUFLAGS) backend_cuda.cu tests/test_mxfp4_cuda.cu tests/mxfp4_ref.o -o mxfp4_cuda_test$(EXE)+ "$(GPUCC)" $(GPUFLAGS) backend_cuda.cu tests/test_mxfp4_cuda.cu tests/mxfp4_ref.o -o mxfp4_cuda_test$(EXE) $(GPU_TEST_LIBS)./mxfp4_cuda_test$(EXE)# Allocator footprint (#687): what a cudaMalloc really takes off the card,# which is what the expert tier has to be charged rather than the logical# byte count. Runs LAST on purpose - it is the only test here that# allocates in bulk to measure an amortised cost, so it should not leave# the card fragmented underneath a kernel test that follows it.- "$(GPUCC)" $(GPUFLAGS) backend_cuda.cu tests/test_alloc_footprint_cuda.cu -o alloc_footprint_test$(EXE)+ "$(GPUCC)" $(GPUFLAGS) backend_cuda.cu tests/test_alloc_footprint_cuda.cu -o alloc_footprint_test$(EXE) $(GPU_TEST_LIBS)./alloc_footprint_test$(EXE)@@ -990,12 +994,12 @@ hip-test:# CI: compile the backend and its test binary WITHOUT executing them (for# runners with the toolchain but no GPU). Pass CUDA_ARCH/HIP=1+HIP_ARCH.gpu-compile: backend_cuda.o- "$(GPUCC)" $(GPUFLAGS) backend_cuda.cu tests/test_backend_cuda.cu -o backend_cuda_test$(EXE) $(ANS_NVCC_LIBS)- "$(GPUCC)" $(GPUFLAGS) backend_cuda.cu tests/test_ragged_attention.cu -o ragged_attention_test$(EXE)- "$(GPUCC)" $(GPUFLAGS) backend_cuda.cu tests/test_absorb_determinism.cu -o absorb_determinism_test$(EXE)- "$(GPUCC)" $(GPUFLAGS) tests/test_fp8_warp_cuda.cu -o fp8_warp_test$(EXE) $(ANS_NVCC_LIBS)- "$(GPUCC)" $(GPUFLAGS) tests/test_weights_owned_cuda.cu -o weights_owned_test$(EXE) $(ANS_NVCC_LIBS)- "$(GPUCC)" $(GPUFLAGS) tests/test_cuda_fmt_trap_cuda.cu -o cuda_fmt_trap_test$(EXE) $(ANS_NVCC_LIBS)+ "$(GPUCC)" $(GPUFLAGS) backend_cuda.cu tests/test_backend_cuda.cu -o backend_cuda_test$(EXE) $(ANS_NVCC_LIBS) $(GPU_TEST_LIBS)+ "$(GPUCC)" $(GPUFLAGS) backend_cuda.cu tests/test_ragged_attention.cu -o ragged_attention_test$(EXE) $(GPU_TEST_LIBS)+ "$(GPUCC)" $(GPUFLAGS) backend_cuda.cu tests/test_absorb_determinism.cu -o absorb_determinism_test$(EXE) $(GPU_TEST_LIBS)+ "$(GPUCC)" $(GPUFLAGS) tests/test_fp8_warp_cuda.cu -o fp8_warp_test$(EXE) $(ANS_NVCC_LIBS) $(GPU_TEST_LIBS)+ "$(GPUCC)" $(GPUFLAGS) tests/test_weights_owned_cuda.cu -o weights_owned_test$(EXE) $(ANS_NVCC_LIBS) $(GPU_TEST_LIBS)+ "$(GPUCC)" $(GPUFLAGS) tests/test_cuda_fmt_trap_cuda.cu -o cuda_fmt_trap_test$(EXE) $(ANS_NVCC_LIBS) $(GPU_TEST_LIBS)cuda-bench: backend_cuda.cu backend_cuda.h backend_gpu_compat.h tests/bench_tensor_core.cu@command -v "$(GPUCC)" >/dev/null 2>&1 || { echo "$(GPUCC_NAME) not found" >&2; exit 1; }diff --git a/c/backend_cuda.cu b/c/backend_cuda.cuindex 14f1b56f..d55f9647 100644--- a/c/backend_cuda.cu+++ b/c/backend_cuda.cu@@ -1903,11 +1903,16 @@ static int expert_group_impl(ColiCudaTensor *const *gates,if(profile) cudaEventRecord(ev[1],ctx->stream);GroupDesc *dev=(GroupDesc*)ctx->group_desc;int tc=getenv("COLI_CUDA_TC_INT4")&&atoi(getenv("COLI_CUDA_TC_INT4"));- /* grouped_s4_wmma's body needs __CUDA_ARCH__>=750: on builds where the- * WMMA kernels are compiled out (COLI_HIP_NO_WMMA) the launch would- * succeed with an EMPTY kernel and the output buffer would silently keep- * stale data. Gate the branch like TC_W4A16 below does. */- tc=tc&&!pin_small_batch&&COLI_GPU_HAS_WMMA&&all_s4&&D%32==0&&I%32==0&&D%8==0&&I%8==0;+ /* grouped_s4_wmma's body needs __CUDA_ARCH__>=750 AND the s4 fragment type:+ * on a build where either is missing (COLI_HIP_NO_WMMA, or rocWMMA, which+ * has no sub-byte precision and defines __CUDA_ARCH__ 700) the launch+ * succeeds with an EMPTY kernel and the output buffer silently keeps the+ * previous call's data (#1499). So the gate mirrors the body's own guard:+ * the s4 flag from backend_gpu_compat.h plus the device's compute+ * capability, not WMMA availability alone. */+ tc=tc&&!pin_small_batch&&COLI_GPU_HAS_WMMA&&COLI_GPU_HAS_S4_WMMA&&+ (ctx->compute_major*10+ctx->compute_minor>=75)&&+ all_s4&&D%32==0&&I%32==0&&D%8==0&&I%8==0;int tc_min=getenv("COLI_CUDA_TC_MIN_ROWS")?atoi(getenv("COLI_CUDA_TC_MIN_ROWS")):8;for(int c=0;c<count&&tc;c++)tc=rows[c]>=tc_min;if(all_e8){diff --git a/c/backend_gpu_compat.h b/c/backend_gpu_compat.hindex f30a60ed..dbbd07ff 100644--- a/c/backend_gpu_compat.h+++ b/c/backend_gpu_compat.h@@ -36,6 +36,13 @@#if __has_include(<rocwmma/rocwmma.hpp>)#include <rocwmma/rocwmma.hpp>#define COLI_GPU_HAS_WMMA 1+/* rocWMMA has no sub-byte fragments (no s4/int4 precision), and the+ * grouped_s4_wmma body is guarded by __CUDA_ARCH__ >= 750, which the 700+ * defined below does not reach: its body is compiled OUT. The host gate+ * used to test rocWMMA availability alone, took the branch, launched three+ * empty kernels and returned the previous call's output untouched (#1499).+ * A second flag for the one kernel that needs more than WMMA. */+#define COLI_GPU_HAS_S4_WMMA 0#define __CUDA_ARCH__ 700#define __half rocwmma::float16_tnamespace nvcuda { namespace wmma = ::rocwmma; }@@ -48,6 +55,7 @@ namespace nvcuda { namespace wmma = ::rocwmma; }#else/* Arch has no matrix cores: rocWMMA is not needed and not included. */#define COLI_GPU_HAS_WMMA 0+#define COLI_GPU_HAS_S4_WMMA 0#define __syncwarp() __syncthreads()#endif#define cudaError_t hipError_t@@ -101,6 +109,10 @@ namespace nvcuda { namespace wmma = ::rocwmma; }#include <cuda_runtime.h>#include <mma.h>#define COLI_GPU_HAS_WMMA 1+/* nvcuda::wmma::experimental::precision::s4 exists here; the kernel body still+ * needs sm_75+, which the host checks at run time from the device's compute+ * capability (see the grouped_s4_wmma dispatch in backend_cuda.cu). */+#define COLI_GPU_HAS_S4_WMMA 1#endif#endif /* COLIBRI_BACKEND_GPU_COMPAT_H */diff --git a/c/tests/test_backend_cuda.cu b/c/tests/test_backend_cuda.cuindex 6a807942..049ccf0a 100644--- a/c/tests/test_backend_cuda.cu+++ b/c/tests/test_backend_cuda.cu@@ -398,6 +398,11 @@ int main(int argc, char **argv) {}setenv("COLI_CUDA_TC_INT4","1",1);setenv("COLI_CUDA_TC_MIN_ROWS","1",1);+ /* #1499: poison the destination first. The device buffer this call writes+ * still held scalar4 from the call above, so an EMPTY tensor-core kernel+ * (rocWMMA, or a card below sm_75 with the body compiled out) left it+ * untouched and the comparison below passed against itself. */+ for(int i=0;i<64;i++)tensor4[i]=-12345.f;if(!coli_cuda_expert_group(gg4,ug4,dg4,group_rows,2,tensor4,gx4)||!relative_rms(tensor4,scalar4,64,0.30f))return 1;if(!coli_cuda_expert_group_pinned(gg4,ug4,dg4,group_rows,2,pinned4,gx4,1)||