Skip to content

Run frontier MoE models on hardware you already own — pure C, zero deps, experts streamed from disk. Tiny engine, immense model. 🐦

C

gpu: gate grouped_s4_wmma on the s4 fragment type and sm_75, not on WMMA

JustVugg19af0fc28dcf
diff --git a/c/Makefile b/c/Makefile
index ba136a48..ae176034 100644
--- a/c/Makefile
+++ b/c/Makefile
@@ -577,6 +577,10 @@ CUDA_OBJ = backend_cuda.o
GPUCC = $(HIPCC)
GPUFLAGS = $(HIPCCFLAGS)
GPUCC_NAME = hipcc (set ROCM_HOME or HIPCC)
+# The kernel test binaries link the HIP runtime explicitly: hipcc does not add
+# it for a plain link the way nvcc adds cudart (#1499: `make hip-test` failed
+# with undefined hipStreamSynchronize and the suite never ran on ROCm).
+GPU_TEST_LIBS = -L$(ROCM_HOME)/lib -lamdhip64
# rocWMMA needs matrix cores (MFMA gfx908+ / WMMA gfx11xx); on these archs its
# headers static_assert (ROCm/TheRock#1944). Compile the WMMA paths out and
# let backend_gpu_compat.h fall back via COLI_GPU_HAS_WMMA=0. The flag must
@@ -897,23 +901,23 @@ $(RANSLIB): tools/rans_ctypes.c rans.h
cuda-test: backend_cuda.cu backend_cuda.h backend_gpu_compat.h tests/test_backend_cuda.cu tests/test_ragged_attention.cu tests/test_absorb_determinism.cu tests/test_mxfp4_cuda.cu tests/mxfp4_ref.c tests/test_fp8_warp_cuda.cu tests/test_fp8_cuda.cu tests/test_weights_owned_cuda.cu tests/test_cuda_fmt_trap_cuda.cu tests/test_alloc_footprint_cuda.cu
@command -v "$(GPUCC)" >/dev/null 2>&1 || { echo "$(GPUCC_NAME) not found" >&2; exit 1; }
- "$(GPUCC)" $(GPUFLAGS) backend_cuda.cu tests/test_backend_cuda.cu -o backend_cuda_test$(EXE) $(ANS_NVCC_LIBS)
+ "$(GPUCC)" $(GPUFLAGS) backend_cuda.cu tests/test_backend_cuda.cu -o backend_cuda_test$(EXE) $(ANS_NVCC_LIBS) $(GPU_TEST_LIBS)
./backend_cuda_test$(EXE)
- "$(GPUCC)" $(GPUFLAGS) backend_cuda.cu tests/test_ragged_attention.cu -o ragged_attention_test$(EXE)
+ "$(GPUCC)" $(GPUFLAGS) backend_cuda.cu tests/test_ragged_attention.cu -o ragged_attention_test$(EXE) $(GPU_TEST_LIBS)
./ragged_attention_test$(EXE)
# weight_at's device-side refusal (the __trap()) on real silicon. The
# host-side half is tests/test_cuda_fmt_guard.c, which runs in the CPU
# suite and cannot execute device code. Includes backend_cuda.cu itself:
# weight_at is file-static, and the trap is unreachable through the public
# API by design. Re-execs itself per probe (the trap poisons the context).
- "$(GPUCC)" $(GPUFLAGS) tests/test_cuda_fmt_trap_cuda.cu -o cuda_fmt_trap_test$(EXE) $(ANS_NVCC_LIBS)
+ "$(GPUCC)" $(GPUFLAGS) tests/test_cuda_fmt_trap_cuda.cu -o cuda_fmt_trap_test$(EXE) $(ANS_NVCC_LIBS) $(GPU_TEST_LIBS)
./cuda_fmt_trap_test$(EXE)
# fmt=8 warp kernels (COLI_CUDA_F8_WARP): decode sweep, census + tail
# shapes, S-invariance, NaN/denormal policy. Includes backend_cuda.cu
# itself (kernel-level oracle), so only the test file is compiled.
# Runs BEFORE the MXFP4 test: that one has a pre-existing exponent-255
# failure on some hosts and would otherwise abort the recipe first.
- "$(GPUCC)" $(GPUFLAGS) tests/test_fp8_warp_cuda.cu -o fp8_warp_test$(EXE) $(ANS_NVCC_LIBS)
+ "$(GPUCC)" $(GPUFLAGS) tests/test_fp8_warp_cuda.cu -o fp8_warp_test$(EXE) $(ANS_NVCC_LIBS) $(GPU_TEST_LIBS)
./fp8_warp_test$(EXE)
# The batch and ragged absorb kernels must reproduce their own output: they
# reduce in a fixed tree order with no atomics, so two identical launches
@@ -925,18 +929,18 @@ cuda-test: backend_cuda.cu backend_cuda.h backend_gpu_compat.h tests/test_backen
# test in the recipe, and it should not cost the broader kernel suites if
# it trips. It stays BEFORE MXFP4 so it still runs on the hosts where that
# one is expected to fail.
- "$(GPUCC)" $(GPUFLAGS) backend_cuda.cu tests/test_absorb_determinism.cu -o absorb_determinism_test$(EXE)
+ "$(GPUCC)" $(GPUFLAGS) backend_cuda.cu tests/test_absorb_determinism.cu -o absorb_determinism_test$(EXE) $(GPU_TEST_LIBS)
./absorb_determinism_test$(EXE)
# fmt=8 (fp8-e4m3) kernel oracle + public API (LUT gate, dense, expert
# group, byte accounting). Includes backend_cuda.cu itself, so only the
# test file is compiled. Also runs BEFORE the MXFP4 test (same reason).
- "$(GPUCC)" $(GPUFLAGS) tests/test_fp8_cuda.cu -o fp8_cuda_test$(EXE) $(ANS_NVCC_LIBS)
+ "$(GPUCC)" $(GPUFLAGS) tests/test_fp8_cuda.cu -o fp8_cuda_test$(EXE) $(ANS_NVCC_LIBS) $(GPU_TEST_LIBS)
./fp8_cuda_test$(EXE)
# Upload-failure ownership: a failed H2D weight copy must release the
# bytes it allocated (weights_owned set by the malloc, not the copy).
# Includes backend_cuda.cu itself to plant the memcpy failure, so only
# the test file is compiled.
- "$(GPUCC)" $(GPUFLAGS) tests/test_weights_owned_cuda.cu -o weights_owned_test$(EXE) $(ANS_NVCC_LIBS)
+ "$(GPUCC)" $(GPUFLAGS) tests/test_weights_owned_cuda.cu -o weights_owned_test$(EXE) $(ANS_NVCC_LIBS) $(GPU_TEST_LIBS)
./weights_owned_test$(EXE)
# MXFP4 (fmt=7) against the CPU decoder in quant.h. The reference is built
# as C on purpose: quant.h uses _Thread_local, which nvcc's C++ front end
@@ -955,14 +959,14 @@ cuda-test: backend_cuda.cu backend_cuda.h backend_gpu_compat.h tests/test_backen
# below no longer pulls the runtime, and CI could not catch it because CI
# never runs cuda-test. -Xclang appears nowhere else in CFLAGS.
$(CC) $(filter-out -Xclang -fopenmp,$(CFLAGS)) -fno-openmp -c tests/mxfp4_ref.c -o tests/mxfp4_ref.o
- "$(GPUCC)" $(GPUFLAGS) backend_cuda.cu tests/test_mxfp4_cuda.cu tests/mxfp4_ref.o -o mxfp4_cuda_test$(EXE)
+ "$(GPUCC)" $(GPUFLAGS) backend_cuda.cu tests/test_mxfp4_cuda.cu tests/mxfp4_ref.o -o mxfp4_cuda_test$(EXE) $(GPU_TEST_LIBS)
./mxfp4_cuda_test$(EXE)
# Allocator footprint (#687): what a cudaMalloc really takes off the card,
# which is what the expert tier has to be charged rather than the logical
# byte count. Runs LAST on purpose - it is the only test here that
# allocates in bulk to measure an amortised cost, so it should not leave
# the card fragmented underneath a kernel test that follows it.
- "$(GPUCC)" $(GPUFLAGS) backend_cuda.cu tests/test_alloc_footprint_cuda.cu -o alloc_footprint_test$(EXE)
+ "$(GPUCC)" $(GPUFLAGS) backend_cuda.cu tests/test_alloc_footprint_cuda.cu -o alloc_footprint_test$(EXE) $(GPU_TEST_LIBS)
./alloc_footprint_test$(EXE)
@@ -990,12 +994,12 @@ hip-test:
# CI: compile the backend and its test binary WITHOUT executing them (for
# runners with the toolchain but no GPU). Pass CUDA_ARCH/HIP=1+HIP_ARCH.
gpu-compile: backend_cuda.o
- "$(GPUCC)" $(GPUFLAGS) backend_cuda.cu tests/test_backend_cuda.cu -o backend_cuda_test$(EXE) $(ANS_NVCC_LIBS)
- "$(GPUCC)" $(GPUFLAGS) backend_cuda.cu tests/test_ragged_attention.cu -o ragged_attention_test$(EXE)
- "$(GPUCC)" $(GPUFLAGS) backend_cuda.cu tests/test_absorb_determinism.cu -o absorb_determinism_test$(EXE)
- "$(GPUCC)" $(GPUFLAGS) tests/test_fp8_warp_cuda.cu -o fp8_warp_test$(EXE) $(ANS_NVCC_LIBS)
- "$(GPUCC)" $(GPUFLAGS) tests/test_weights_owned_cuda.cu -o weights_owned_test$(EXE) $(ANS_NVCC_LIBS)
- "$(GPUCC)" $(GPUFLAGS) tests/test_cuda_fmt_trap_cuda.cu -o cuda_fmt_trap_test$(EXE) $(ANS_NVCC_LIBS)
+ "$(GPUCC)" $(GPUFLAGS) backend_cuda.cu tests/test_backend_cuda.cu -o backend_cuda_test$(EXE) $(ANS_NVCC_LIBS) $(GPU_TEST_LIBS)
+ "$(GPUCC)" $(GPUFLAGS) backend_cuda.cu tests/test_ragged_attention.cu -o ragged_attention_test$(EXE) $(GPU_TEST_LIBS)
+ "$(GPUCC)" $(GPUFLAGS) backend_cuda.cu tests/test_absorb_determinism.cu -o absorb_determinism_test$(EXE) $(GPU_TEST_LIBS)
+ "$(GPUCC)" $(GPUFLAGS) tests/test_fp8_warp_cuda.cu -o fp8_warp_test$(EXE) $(ANS_NVCC_LIBS) $(GPU_TEST_LIBS)
+ "$(GPUCC)" $(GPUFLAGS) tests/test_weights_owned_cuda.cu -o weights_owned_test$(EXE) $(ANS_NVCC_LIBS) $(GPU_TEST_LIBS)
+ "$(GPUCC)" $(GPUFLAGS) tests/test_cuda_fmt_trap_cuda.cu -o cuda_fmt_trap_test$(EXE) $(ANS_NVCC_LIBS) $(GPU_TEST_LIBS)
cuda-bench: backend_cuda.cu backend_cuda.h backend_gpu_compat.h tests/bench_tensor_core.cu
@command -v "$(GPUCC)" >/dev/null 2>&1 || { echo "$(GPUCC_NAME) not found" >&2; exit 1; }
diff --git a/c/backend_cuda.cu b/c/backend_cuda.cu
index 14f1b56f..d55f9647 100644
--- a/c/backend_cuda.cu
+++ b/c/backend_cuda.cu
@@ -1903,11 +1903,16 @@ static int expert_group_impl(ColiCudaTensor *const *gates,
if(profile) cudaEventRecord(ev[1],ctx->stream);
GroupDesc *dev=(GroupDesc*)ctx->group_desc;
int tc=getenv("COLI_CUDA_TC_INT4")&&atoi(getenv("COLI_CUDA_TC_INT4"));
- /* grouped_s4_wmma's body needs __CUDA_ARCH__>=750: on builds where the
- * WMMA kernels are compiled out (COLI_HIP_NO_WMMA) the launch would
- * succeed with an EMPTY kernel and the output buffer would silently keep
- * stale data. Gate the branch like TC_W4A16 below does. */
- tc=tc&&!pin_small_batch&&COLI_GPU_HAS_WMMA&&all_s4&&D%32==0&&I%32==0&&D%8==0&&I%8==0;
+ /* grouped_s4_wmma's body needs __CUDA_ARCH__>=750 AND the s4 fragment type:
+ * on a build where either is missing (COLI_HIP_NO_WMMA, or rocWMMA, which
+ * has no sub-byte precision and defines __CUDA_ARCH__ 700) the launch
+ * succeeds with an EMPTY kernel and the output buffer silently keeps the
+ * previous call's data (#1499). So the gate mirrors the body's own guard:
+ * the s4 flag from backend_gpu_compat.h plus the device's compute
+ * capability, not WMMA availability alone. */
+ tc=tc&&!pin_small_batch&&COLI_GPU_HAS_WMMA&&COLI_GPU_HAS_S4_WMMA&&
+ (ctx->compute_major*10+ctx->compute_minor>=75)&&
+ all_s4&&D%32==0&&I%32==0&&D%8==0&&I%8==0;
int tc_min=getenv("COLI_CUDA_TC_MIN_ROWS")?atoi(getenv("COLI_CUDA_TC_MIN_ROWS")):8;
for(int c=0;c<count&&tc;c++)tc=rows[c]>=tc_min;
if(all_e8){
diff --git a/c/backend_gpu_compat.h b/c/backend_gpu_compat.h
index f30a60ed..dbbd07ff 100644
--- a/c/backend_gpu_compat.h
+++ b/c/backend_gpu_compat.h
@@ -36,6 +36,13 @@
#if __has_include(<rocwmma/rocwmma.hpp>)
#include <rocwmma/rocwmma.hpp>
#define COLI_GPU_HAS_WMMA 1
+/* rocWMMA has no sub-byte fragments (no s4/int4 precision), and the
+ * grouped_s4_wmma body is guarded by __CUDA_ARCH__ >= 750, which the 700
+ * defined below does not reach: its body is compiled OUT. The host gate
+ * used to test rocWMMA availability alone, took the branch, launched three
+ * empty kernels and returned the previous call's output untouched (#1499).
+ * A second flag for the one kernel that needs more than WMMA. */
+#define COLI_GPU_HAS_S4_WMMA 0
#define __CUDA_ARCH__ 700
#define __half rocwmma::float16_t
namespace nvcuda { namespace wmma = ::rocwmma; }
@@ -48,6 +55,7 @@ namespace nvcuda { namespace wmma = ::rocwmma; }
#else
/* Arch has no matrix cores: rocWMMA is not needed and not included. */
#define COLI_GPU_HAS_WMMA 0
+#define COLI_GPU_HAS_S4_WMMA 0
#define __syncwarp() __syncthreads()
#endif
#define cudaError_t hipError_t
@@ -101,6 +109,10 @@ namespace nvcuda { namespace wmma = ::rocwmma; }
#include <cuda_runtime.h>
#include <mma.h>
#define COLI_GPU_HAS_WMMA 1
+/* nvcuda::wmma::experimental::precision::s4 exists here; the kernel body still
+ * needs sm_75+, which the host checks at run time from the device's compute
+ * capability (see the grouped_s4_wmma dispatch in backend_cuda.cu). */
+#define COLI_GPU_HAS_S4_WMMA 1
#endif
#endif /* COLIBRI_BACKEND_GPU_COMPAT_H */
diff --git a/c/tests/test_backend_cuda.cu b/c/tests/test_backend_cuda.cu
index 6a807942..049ccf0a 100644
--- a/c/tests/test_backend_cuda.cu
+++ b/c/tests/test_backend_cuda.cu
@@ -398,6 +398,11 @@ int main(int argc, char **argv) {
}
setenv("COLI_CUDA_TC_INT4","1",1);
setenv("COLI_CUDA_TC_MIN_ROWS","1",1);
+ /* #1499: poison the destination first. The device buffer this call writes
+ * still held scalar4 from the call above, so an EMPTY tensor-core kernel
+ * (rocWMMA, or a card below sm_75 with the body compiled out) left it
+ * untouched and the comparison below passed against itself. */
+ for(int i=0;i<64;i++)tensor4[i]=-12345.f;
if(!coli_cuda_expert_group(gg4,ug4,dg4,group_rows,2,tensor4,gx4)||
!relative_rms(tensor4,scalar4,64,0.30f))return 1;
if(!coli_cuda_expert_group_pinned(gg4,ug4,dg4,group_rows,2,pinned4,gx4,1)||