Skip to content

Run frontier MoE models on hardware you already own β€” pure C, zero deps, experts streamed from disk. Tiny engine, immense model. 🐦

C

feat: enable rocWMMA tensor cores for AMD HIP backend

Grecco00561ef4e96e
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 426ea413..84278c10 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -58,7 +58,7 @@ jobs:
# link of the test binary below covers the single-source claim, and
# the engine link is exercised on real ROCm hardware (GPU_BACKENDS.md).
docker run --rm -v "$PWD":/src -w /src rocm/dev-ubuntu-24.04:6.2 bash -c '
- apt-get update && apt-get install -y --no-install-recommends make gcc g++ &&
+ apt-get update && apt-get install -y --no-install-recommends make gcc g++ rocwmma-dev &&
make -C c gpu-compile HIP=1 HIP_ARCH=gfx1100'
echo "HIP syntax check passed"
diff --git a/.gitignore b/.gitignore
index f62b3792..05cc9b89 100644
--- a/.gitignore
+++ b/.gitignore
@@ -68,3 +68,7 @@ olmoe_merged/
olmoe_i4/
c/olmoe_merged/
c/olmoe_i4/
+
+# planning docs β€” internal, not part of the project
+ai_plans/
+PR_DESCRIPTION.md
diff --git a/c/Makefile b/c/Makefile
index a598d923..48bdca34 100644
--- a/c/Makefile
+++ b/c/Makefile
@@ -183,7 +183,7 @@ HIP ?= 0
ROCM_HOME ?= /opt/rocm
HIPCC ?= $(ROCM_HOME)/bin/hipcc
HIP_ARCH ?= native
-HIPCCFLAGS ?= -O3 -std=c++17 -x hip --offload-arch=$(HIP_ARCH) -Wall -Wextra
+HIPCCFLAGS ?= -O3 -std=c++17 -x hip --offload-arch=$(HIP_ARCH) -Wall -Wextra -fPIE
GPUCC = $(NVCC)
GPUFLAGS = $(NVCCFLAGS)
GPUCC_NAME = nvcc (set CUDA_HOME or NVCC)
diff --git a/c/backend_gpu_compat.h b/c/backend_gpu_compat.h
index 990462af..91b9f285 100644
--- a/c/backend_gpu_compat.h
+++ b/c/backend_gpu_compat.h
@@ -8,17 +8,32 @@
*
* COLI_GPU_HAS_WMMA: the WMMA tensor-core kernels are guarded by
* __CUDA_ARCH__ >= 700 (device side) and by this flag at the host dispatch
- * sites. Under HIP the flag is 0: gfx GPUs report compute_major >= 7, so a
- * runtime-only check would select empty kernel bodies. Matrix-core support
- * via rocWMMA is a possible follow-up; until then HIP always uses the
- * portable kernels. */
+ * sites. Under HIP, __CUDA_ARCH__ is not defined by hipcc, so we define it
+ * here to 700 to activate the WMMA kernel bodies (the guard is a minimum
+ * threshold; 700 satisfies >= 700). Note: the grouped_s4_wmma kernel has a
+ * stricter guard of __CUDA_ARCH__ >= 750, so it stays a no-op even with
+ * __CUDA_ARCH__=700 β€” see HIP-KNOWN-BUGS.md BUG-002.
+ * rocWMMA provides float16_t and the fragment/mma_sync API; the CUDA
+ * __half and nvcuda::wmma names are mapped to their rocWMMA equivalents.
+ * Note: grouped_s4_wmma uses wmma::experimental::precision::s4 (4-bit)
+ * which has no rocWMMA equivalent β€” that kernel stays a no-op under HIP
+ * and falls back to quant_matmul. */
#ifndef COLIBRI_BACKEND_GPU_COMPAT_H
#define COLIBRI_BACKEND_GPU_COMPAT_H
#if defined(__HIP_PLATFORM_AMD__) || defined(__HIP__)
#include <hip/hip_runtime.h>
#include <hip/hip_fp16.h>
-#define COLI_GPU_HAS_WMMA 0
+#if __has_include(<rocwmma/rocwmma.hpp>)
+#include <rocwmma/rocwmma.hpp>
+#define COLI_GPU_HAS_WMMA 1
+#define __CUDA_ARCH__ 700
+#define __half rocwmma::float16_t
+namespace nvcuda { namespace wmma = ::rocwmma; }
+#define __syncwarp() __syncthreads()
+#else
+#error "rocWMMA headers not found. Install rocwmma-dev (or rocm-hip-runtime-dev) to build with HIP."
+#endif
#define cudaError_t hipError_t
#define cudaSuccess hipSuccess
#define cudaGetErrorString hipGetErrorString
@@ -63,4 +78,4 @@
#define COLI_GPU_HAS_WMMA 1
#endif
-#endif
+#endif /* COLIBRI_BACKEND_GPU_COMPAT_H */
diff --git a/c/coli b/c/coli
index 5864e689..ad0343a7 100755
--- a/c/coli
+++ b/c/coli
@@ -154,7 +154,8 @@ def cuda_binary():
if sys.platform == "linux":
try:
linked=subprocess.run(["ldd",GLM],capture_output=True,text=True,timeout=3)
- return any("libcudart" in line and "not found" not in line
+ # Detect both CUDA (libcudart) and HIP (libamdhip64) builds.
+ return any(("libcudart" in line or "libamdhip64" in line) and "not found" not in line
for line in linked.stdout.splitlines())
except (OSError,subprocess.SubprocessError): return False
if sys.platform == "win32":