Run frontier MoE models on hardware you already own — pure C, zero deps, experts streamed from disk. Tiny engine, immense model. 🐦
C
fix(hip): support GPUs without matrix cores (gfx9xx, gfx101x/103x)
Mr. Metric0ff5bb8b0e3d
diff --git a/c/Makefile b/c/Makefileindex 9247ab81..71b4fd30 100644--- a/c/Makefile+++ b/c/Makefile@@ -183,7 +183,22 @@ HIP ?= 0ROCM_HOME ?= /opt/rocmHIPCC ?= $(ROCM_HOME)/bin/hipccHIP_ARCH ?= native-HIPCCFLAGS ?= -O3 -std=c++17 -x hip --offload-arch=$(HIP_ARCH) -Wall -Wextra -fPIE+# HIP_ARCH accepts a list (space- or comma-separated, e.g. "gfx906 gfx1030")+# to fat-bundle several targets into one binary; COLI_GPUS picks at runtime.+# `native` is resolved to the local GPUs' arch names HERE (not left to hipcc):+# the NO_WMMA_ARCHS filter below must see real gfx names to know whether the+# rocWMMA paths can compile at all. `override` beats command-line assignment.+comma := ,+override HIP_ARCH := $(subst $(comma), ,$(HIP_ARCH))+ifeq ($(strip $(HIP_ARCH)),native)+ifeq ($(HIP),1)+override HIP_ARCH := $(shell $(ROCM_HOME)/bin/rocm_agent_enumerator 2>/dev/null | grep -v gfx000 | sort -u)+ifeq ($(strip $(HIP_ARCH)),)+$(error HIP_ARCH=native but rocm_agent_enumerator found no GPUs; set HIP_ARCH=gfxNNNN explicitly)+endif+endif+endif+HIPCCFLAGS ?= -O3 -std=c++17 -x hip $(foreach a,$(HIP_ARCH),--offload-arch=$(a)) -Wall -Wextra -fPIEGPUCC = $(NVCC)GPUFLAGS = $(NVCCFLAGS)GPUCC_NAME = nvcc (set CUDA_HOME or NVCC)@@ -240,6 +255,23 @@ CUDA_OBJ = backend_cuda.oGPUCC = $(HIPCC)GPUFLAGS = $(HIPCCFLAGS)GPUCC_NAME = hipcc (set ROCM_HOME or HIPCC)+# rocWMMA needs matrix cores (MFMA gfx908+ / WMMA gfx11xx); on these archs its+# headers static_assert (ROCm/TheRock#1944). Compile the WMMA paths out and+# let backend_gpu_compat.h fall back via COLI_GPU_HAS_WMMA=0. The flag must+# reach both hipcc passes, hence a compile define, not an arch macro. If a+# HIP_ARCH=native build dies inside rocwmma, set the arch explicitly.+NO_WMMA_ARCHS = gfx803 gfx900 gfx902 gfx904 gfx906 gfx90c \+ gfx1010 gfx1011 gfx1012 gfx1013 \+ gfx1030 gfx1031 gfx1032 gfx1033 gfx1034 gfx1035 gfx1036 gfx1103+ifneq (,$(filter $(HIP_ARCH),$(NO_WMMA_ARCHS)))+HIPCCFLAGS += -DCOLI_HIP_NO_WMMA+# The define must hold for every arch in the fat binary (host dispatch is+# shared), so one incapable arch disables the WMMA kernels for all of them.+# Loud, because on a WMMA-capable card that is a real capability loss.+ifneq (,$(filter-out $(NO_WMMA_ARCHS),$(HIP_ARCH)))+$(warning mixed HIP_ARCH list: $(filter $(HIP_ARCH),$(NO_WMMA_ARCHS)) cannot run rocWMMA -> tensor-core kernels disabled for ALL targets ($(HIP_ARCH)). Build separate binaries to keep WMMA on the capable card.)+endif+endifendif# METAL=1 adds an opt-in Apple-GPU backend (macOS only). The shader is compiled atdiff --git a/c/backend_cuda.cu b/c/backend_cuda.cuindex 2ff4eb8a..6642eb96 100644--- a/c/backend_cuda.cu+++ b/c/backend_cuda.cu@@ -976,7 +976,11 @@ extern "C" int coli_cuda_expert_group(ColiCudaTensor *const *gates,if(profile) cudaEventRecord(ev[1],ctx->stream);GroupDesc *dev=(GroupDesc*)ctx->group_desc;int tc=getenv("COLI_CUDA_TC_INT4")&&atoi(getenv("COLI_CUDA_TC_INT4"));- tc=tc&&all_s4&&D%32==0&&I%32==0&&D%8==0&&I%8==0;+ /* grouped_s4_wmma's body needs __CUDA_ARCH__>=750: on builds where the+ * WMMA kernels are compiled out (COLI_HIP_NO_WMMA) the launch would+ * succeed with an EMPTY kernel and the output buffer would silently keep+ * stale data. Gate the branch like TC_W4A16 below does. */+ tc=tc&&COLI_GPU_HAS_WMMA&&all_s4&&D%32==0&&I%32==0&&D%8==0&&I%8==0;int tc_min=getenv("COLI_CUDA_TC_MIN_ROWS")?atoi(getenv("COLI_CUDA_TC_MIN_ROWS")):8;for(int c=0;c<count&&tc;c++)tc=rows[c]>=tc_min;if(tc){diff --git a/c/backend_gpu_compat.h b/c/backend_gpu_compat.hindex 74bbd578..7f22863f 100644--- a/c/backend_gpu_compat.h+++ b/c/backend_gpu_compat.h@@ -24,6 +24,15 @@#if defined(__HIP_PLATFORM_AMD__) || defined(__HIP__)#include <hip/hip_runtime.h>#include <hip/hip_fp16.h>+/* rocWMMA requires matrix cores (MFMA: gfx908+, WMMA: gfx11xx); on other+ * targets (gfx906, gfx101x, gfx103x) its headers static_assert. The Makefile+ * passes -DCOLI_HIP_NO_WMMA for those archs: the WMMA kernel bodies stay+ * compiled out (__CUDA_ARCH__ undefined fails the >= 700 guard) and the host+ * dispatch sites fall back via COLI_GPU_HAS_WMMA=0, same as pre-Volta CUDA.+ * The flag must be set for BOTH hipcc passes (host + device) — an arch-macro+ * check here would let the host dispatch kernels whose bodies were compiled+ * out, silently computing nothing. */+#ifndef COLI_HIP_NO_WMMA#if __has_include(<rocwmma/rocwmma.hpp>)#include <rocwmma/rocwmma.hpp>#define COLI_GPU_HAS_WMMA 1@@ -32,7 +41,14 @@namespace nvcuda { namespace wmma = ::rocwmma; }#define __syncwarp() __syncthreads()#else-#error "rocWMMA headers not found. Install rocwmma-dev (or rocm-hip-runtime-dev) to build with HIP."+/* WMMA-capable arch but no headers: stop loudly rather than silently build+ * a binary with the tensor-core kernels disabled. */+#error "rocWMMA headers not found. Install rocwmma-dev (or rocm-hip-runtime-dev), or build for a non-WMMA arch (see NO_WMMA_ARCHS in the Makefile)."+#endif+#else+/* Arch has no matrix cores: rocWMMA is not needed and not included. */+#define COLI_GPU_HAS_WMMA 0+#define __syncwarp() __syncthreads()#endif#define cudaError_t hipError_t#define cudaSuccess hipSuccess