Skip to content

Run frontier MoE models on hardware you already own — pure C, zero deps, experts streamed from disk. Tiny engine, immense model. 🐦

C

fix(hip): support GPUs without matrix cores (gfx9xx, gfx101x/103x)

Mr. Metric0ff5bb8b0e3d
diff --git a/c/Makefile b/c/Makefile
index 9247ab81..71b4fd30 100644
--- a/c/Makefile
+++ b/c/Makefile
@@ -183,7 +183,22 @@ HIP ?= 0
ROCM_HOME ?= /opt/rocm
HIPCC ?= $(ROCM_HOME)/bin/hipcc
HIP_ARCH ?= native
-HIPCCFLAGS ?= -O3 -std=c++17 -x hip --offload-arch=$(HIP_ARCH) -Wall -Wextra -fPIE
+# HIP_ARCH accepts a list (space- or comma-separated, e.g. "gfx906 gfx1030")
+# to fat-bundle several targets into one binary; COLI_GPUS picks at runtime.
+# `native` is resolved to the local GPUs' arch names HERE (not left to hipcc):
+# the NO_WMMA_ARCHS filter below must see real gfx names to know whether the
+# rocWMMA paths can compile at all. `override` beats command-line assignment.
+comma := ,
+override HIP_ARCH := $(subst $(comma), ,$(HIP_ARCH))
+ifeq ($(strip $(HIP_ARCH)),native)
+ifeq ($(HIP),1)
+override HIP_ARCH := $(shell $(ROCM_HOME)/bin/rocm_agent_enumerator 2>/dev/null | grep -v gfx000 | sort -u)
+ifeq ($(strip $(HIP_ARCH)),)
+$(error HIP_ARCH=native but rocm_agent_enumerator found no GPUs; set HIP_ARCH=gfxNNNN explicitly)
+endif
+endif
+endif
+HIPCCFLAGS ?= -O3 -std=c++17 -x hip $(foreach a,$(HIP_ARCH),--offload-arch=$(a)) -Wall -Wextra -fPIE
GPUCC = $(NVCC)
GPUFLAGS = $(NVCCFLAGS)
GPUCC_NAME = nvcc (set CUDA_HOME or NVCC)
@@ -240,6 +255,23 @@ CUDA_OBJ = backend_cuda.o
GPUCC = $(HIPCC)
GPUFLAGS = $(HIPCCFLAGS)
GPUCC_NAME = hipcc (set ROCM_HOME or HIPCC)
+# rocWMMA needs matrix cores (MFMA gfx908+ / WMMA gfx11xx); on these archs its
+# headers static_assert (ROCm/TheRock#1944). Compile the WMMA paths out and
+# let backend_gpu_compat.h fall back via COLI_GPU_HAS_WMMA=0. The flag must
+# reach both hipcc passes, hence a compile define, not an arch macro. If a
+# HIP_ARCH=native build dies inside rocwmma, set the arch explicitly.
+NO_WMMA_ARCHS = gfx803 gfx900 gfx902 gfx904 gfx906 gfx90c \
+ gfx1010 gfx1011 gfx1012 gfx1013 \
+ gfx1030 gfx1031 gfx1032 gfx1033 gfx1034 gfx1035 gfx1036 gfx1103
+ifneq (,$(filter $(HIP_ARCH),$(NO_WMMA_ARCHS)))
+HIPCCFLAGS += -DCOLI_HIP_NO_WMMA
+# The define must hold for every arch in the fat binary (host dispatch is
+# shared), so one incapable arch disables the WMMA kernels for all of them.
+# Loud, because on a WMMA-capable card that is a real capability loss.
+ifneq (,$(filter-out $(NO_WMMA_ARCHS),$(HIP_ARCH)))
+$(warning mixed HIP_ARCH list: $(filter $(HIP_ARCH),$(NO_WMMA_ARCHS)) cannot run rocWMMA -> tensor-core kernels disabled for ALL targets ($(HIP_ARCH)). Build separate binaries to keep WMMA on the capable card.)
+endif
+endif
endif
# METAL=1 adds an opt-in Apple-GPU backend (macOS only). The shader is compiled at
diff --git a/c/backend_cuda.cu b/c/backend_cuda.cu
index 2ff4eb8a..6642eb96 100644
--- a/c/backend_cuda.cu
+++ b/c/backend_cuda.cu
@@ -976,7 +976,11 @@ extern "C" int coli_cuda_expert_group(ColiCudaTensor *const *gates,
if(profile) cudaEventRecord(ev[1],ctx->stream);
GroupDesc *dev=(GroupDesc*)ctx->group_desc;
int tc=getenv("COLI_CUDA_TC_INT4")&&atoi(getenv("COLI_CUDA_TC_INT4"));
- tc=tc&&all_s4&&D%32==0&&I%32==0&&D%8==0&&I%8==0;
+ /* grouped_s4_wmma's body needs __CUDA_ARCH__>=750: on builds where the
+ * WMMA kernels are compiled out (COLI_HIP_NO_WMMA) the launch would
+ * succeed with an EMPTY kernel and the output buffer would silently keep
+ * stale data. Gate the branch like TC_W4A16 below does. */
+ tc=tc&&COLI_GPU_HAS_WMMA&&all_s4&&D%32==0&&I%32==0&&D%8==0&&I%8==0;
int tc_min=getenv("COLI_CUDA_TC_MIN_ROWS")?atoi(getenv("COLI_CUDA_TC_MIN_ROWS")):8;
for(int c=0;c<count&&tc;c++)tc=rows[c]>=tc_min;
if(tc){
diff --git a/c/backend_gpu_compat.h b/c/backend_gpu_compat.h
index 74bbd578..7f22863f 100644
--- a/c/backend_gpu_compat.h
+++ b/c/backend_gpu_compat.h
@@ -24,6 +24,15 @@
#if defined(__HIP_PLATFORM_AMD__) || defined(__HIP__)
#include <hip/hip_runtime.h>
#include <hip/hip_fp16.h>
+/* rocWMMA requires matrix cores (MFMA: gfx908+, WMMA: gfx11xx); on other
+ * targets (gfx906, gfx101x, gfx103x) its headers static_assert. The Makefile
+ * passes -DCOLI_HIP_NO_WMMA for those archs: the WMMA kernel bodies stay
+ * compiled out (__CUDA_ARCH__ undefined fails the >= 700 guard) and the host
+ * dispatch sites fall back via COLI_GPU_HAS_WMMA=0, same as pre-Volta CUDA.
+ * The flag must be set for BOTH hipcc passes (host + device) — an arch-macro
+ * check here would let the host dispatch kernels whose bodies were compiled
+ * out, silently computing nothing. */
+#ifndef COLI_HIP_NO_WMMA
#if __has_include(<rocwmma/rocwmma.hpp>)
#include <rocwmma/rocwmma.hpp>
#define COLI_GPU_HAS_WMMA 1
@@ -32,7 +41,14 @@
namespace nvcuda { namespace wmma = ::rocwmma; }
#define __syncwarp() __syncthreads()
#else
-#error "rocWMMA headers not found. Install rocwmma-dev (or rocm-hip-runtime-dev) to build with HIP."
+/* WMMA-capable arch but no headers: stop loudly rather than silently build
+ * a binary with the tensor-core kernels disabled. */
+#error "rocWMMA headers not found. Install rocwmma-dev (or rocm-hip-runtime-dev), or build for a non-WMMA arch (see NO_WMMA_ARCHS in the Makefile)."
+#endif
+#else
+/* Arch has no matrix cores: rocWMMA is not needed and not included. */
+#define COLI_GPU_HAS_WMMA 0
+#define __syncwarp() __syncthreads()
#endif
#define cudaError_t hipError_t
#define cudaSuccess hipSuccess