Skip to content

FreeToken brings datacenter-scale model serving to your desktop. Run massive models locally, fast and efficiently.

feat(rocm): add RDNA3 and RDNA4 runtime foundation (#132)

ZihaoMu0d652e73a452
diff --git a/docs/install.md b/docs/install.md
index 9438c21a..f474216b 100644
--- a/docs/install.md
+++ b/docs/install.md
@@ -1,5 +1,7 @@
# Install
+AMD users: see the [AMD ROCm installation guide (WIP)](install_amd.md).
+
## Requirements
- Linux x86_64, NVIDIA GPU, driver r580+ (CUDA 13)
diff --git a/docs/install_amd.md b/docs/install_amd.md
new file mode 100644
index 00000000..d8b0c1bc
--- /dev/null
+++ b/docs/install_amd.md
@@ -0,0 +1,39 @@
+# AMD ROCm installation (WIP)
+
+AMD support is experimental and remains a work in progress. See the
+[AMD Support roadmap](https://github.com/FlashML-org/FreeToken/issues/541) for
+the current integration and qualification status.
+
+## Requirements
+
+- Linux x86_64
+- AMD RDNA3/RDNA4 GPU (`gfx1100`-`gfx1103`, `gfx1200`, or `gfx1201`)
+- ROCm 7.14
+- Python >= 3.10
+
+## Install from source
+
+Use an official ROCm PyTorch image whose PyTorch version satisfies the project's
+`torch>=2.11,<2.12` constraint. For RDNA4, the matching ROCm 7.14 image is:
+
+```bash
+VIDEO_GID="$(getent group video | cut -d: -f3)"
+RENDER_GID="$(getent group render | cut -d: -f3)"
+docker run --rm -it \
+ --device=/dev/kfd --device=/dev/dri \
+ --group-add="$VIDEO_GID" --group-add="$RENDER_GID" --ipc=host \
+ --cap-add=SYS_PTRACE --security-opt seccomp=unconfined \
+ -e PYTORCH_ROCM_ARCH=gfx1201 -e FREETOKEN_ROCM_ARCH=gfx1201 \
+ -v "$PWD:/workspace/FreeToken" -w /workspace/FreeToken \
+ rocm/pytorch:rocm7.14_ubuntu24.04_py3.12_pytorch_release_2.11.0 bash
+```
+
+Inside the container, preserve the ROCm-enabled PyTorch already supplied by the
+image and disable build isolation so it is also used to compile the extensions:
+
+```bash
+python -m pip install --no-build-isolation -e .
+```
+
+Set both architecture variables to `gfx1200` for RX 9060 family GPUs, or to the
+actual target reported by `rocminfo`.
diff --git a/pyproject.toml b/pyproject.toml
index 25cb204b..8d00d2bb 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -23,6 +23,7 @@ classifiers = [
"Intended Audience :: Science/Research",
"Operating System :: POSIX :: Linux",
"Environment :: GPU :: NVIDIA CUDA",
+ "Environment :: GPU :: AMD ROCm",
"Programming Language :: Python :: 3",
"Programming Language :: Python :: 3.10",
"Programming Language :: Python :: 3.11",
@@ -59,7 +60,10 @@ dependencies = [
"torchvision>=0.26,<0.27",
"tqdm>=4.66,<5",
"transformers>=5.16,<5.17",
- "triton==3.6.0; platform_system == 'Linux'",
+ # CUDA torch 2.11 resolves Triton 3.6; AMD's ROCm 7.14 image supplies its
+ # gfx1201-enabled Triton 3.7 build. Keep both supported without replacing the
+ # runtime-specific wheel selected by the PyTorch distribution.
+ "triton>=3.6,<3.8; platform_system == 'Linux'",
"uvicorn>=0.30,<1",
]
diff --git a/python/freetoken/kernel/__main__.py b/python/freetoken/kernel/__main__.py
index 7be541a6..5b66d484 100644
--- a/python/freetoken/kernel/__main__.py
+++ b/python/freetoken/kernel/__main__.py
@@ -6,27 +6,32 @@ def generate_clangd():
import subprocess
from freetoken.kernel.utils import DEFAULT_INCLUDE
- from freetoken.utils import init_logger
+ from freetoken.utils import get_rocm_gfx_arch, init_logger, is_rocm
from tvm_ffi.libinfo import find_dlpack_include_path, find_include_path
logger = init_logger(__name__)
logger.info("Generating .clangd file...")
include_paths = [find_include_path(), find_dlpack_include_path()] + DEFAULT_INCLUDE
- status = subprocess.run(
- args=["nvidia-smi", "--query-gpu=compute_cap", "--format=csv,noheader"],
- capture_output=True,
- check=True,
- )
- compute_cap = status.stdout.decode("utf-8").strip().split("\n")[0]
- major, minor = compute_cap.split(".")
+
+ # TODO(ROCm): hiprtc JIT cache should be separate from nvcc JIT cache to avoid stale binaries.
+ if is_rocm():
+ arch_flags = ["-xhip", f"--offload-arch={get_rocm_gfx_arch() or 'gfx1201'}"]
+ else:
+ try:
+ status = subprocess.run(
+ args=["nvidia-smi", "--query-gpu=compute_cap", "--format=csv,noheader"],
+ capture_output=True,
+ check=True,
+ )
+ compute_cap = status.stdout.decode("utf-8").strip().split("\n")[0]
+ major, minor = compute_cap.split(".")
+ except (subprocess.CalledProcessError, FileNotFoundError, ValueError):
+ import torch
+
+ major, minor = torch.cuda.get_device_capability()
+ arch_flags = ["-xcuda", f"--cuda-gpu-arch=sm_{major}{minor}"]
compile_flags = ",\n ".join(
- [
- "-xcuda",
- f"--cuda-gpu-arch=sm_{major}{minor}",
- "-std=c++20",
- "-Wall",
- "-Wextra",
- ]
+ arch_flags + ["-std=c++20", "-Wall", "-Wextra"]
+ [f"-isystem{path}" for path in include_paths]
)
clangd_content = f"""
diff --git a/python/freetoken/kernel/_toolchain.py b/python/freetoken/kernel/_toolchain.py
index b49cebbb..93d03221 100644
--- a/python/freetoken/kernel/_toolchain.py
+++ b/python/freetoken/kernel/_toolchain.py
@@ -1,4 +1,4 @@
-"""CUDA toolchain/torch consistency checks.
+"""CUDA/HIP toolchain/torch consistency checks.
Standalone on purpose: setup.py and the kernel-cache build backend load this
file by path, so it must not import the freetoken package.
@@ -16,6 +16,11 @@ ALLOW_MISMATCH_ENV = "FREETOKEN_ALLOW_CUDA_MISMATCH"
_TRUE_VALUES = {"1", "true", "yes", "on"}
+def _is_rocm() -> bool:
+ import torch
+ return getattr(torch.version, "hip", None) is not None
+
+
def _nvcc_path() -> str | None:
from torch.utils.cpp_extension import CUDA_HOME
@@ -49,6 +54,8 @@ def check_nvcc_matches_torch() -> None:
nvcc-built binaries link libcudart.so.<nvcc major>; at runtime only the
torch wheel's own CUDA runtime is guaranteed to be loadable.
"""
+ if _is_rocm():
+ return # ROCm uses hipcc, not nvcc
if os.getenv(ALLOW_MISMATCH_ENV, "").strip().lower() in _TRUE_VALUES:
return
torch_major = torch_cuda_major()
diff --git a/python/freetoken/kernel/backend.py b/python/freetoken/kernel/backend.py
index 2a886cf7..6d1b4a9f 100644
--- a/python/freetoken/kernel/backend.py
+++ b/python/freetoken/kernel/backend.py
@@ -47,6 +47,24 @@ def device_capability() -> tuple[int, int]:
return (int(major), int(minor))
+def is_rocm() -> bool:
+ """True when torch is built for ROCm (AMD GPU)."""
+ import torch
+ return getattr(torch.version, "hip", None) is not None
+
+
+def driver_hip_version() -> int | None:
+ """ROCm driver version, or None if undetermined."""
+ # TODO(ROCm): flashinfer/sgl_kernel have no ROCm builds — Triton fallback is used.
+ try:
+ from freetoken.kernel.pinned import _load_pinned_extension
+ return int(_load_pinned_extension().driver_cuda_version()) or None
+ except Exception:
+ return None
+
+
@functools.cache
def driver_cuda_version() -> int | None:
"""Max CUDA version the installed NVIDIA driver supports (``13000`` == CUDA 13.0),
diff --git a/python/freetoken/kernel/csrc/cpu_moe/cpu_moe_ext.cpp b/python/freetoken/kernel/csrc/cpu_moe/cpu_moe_ext.cpp
index 8349123b..e2cb0d9c 100644
--- a/python/freetoken/kernel/csrc/cpu_moe/cpu_moe_ext.cpp
+++ b/python/freetoken/kernel/csrc/cpu_moe/cpu_moe_ext.cpp
@@ -29,7 +29,7 @@
#include <thread>
#include <vector>
-#include <cuda_runtime_api.h>
+#include <freetoken/hip_compat.h>
#include <torch/extension.h>
#if defined(__linux__)
diff --git a/python/freetoken/kernel/csrc/include/freetoken/hip_compat.h b/python/freetoken/kernel/csrc/include/freetoken/hip_compat.h
new file mode 100644
index 00000000..1898b411
--- /dev/null
+++ b/python/freetoken/kernel/csrc/include/freetoken/hip_compat.h
@@ -0,0 +1,162 @@
+#pragma once
+
+// HIP compatibility shim: maps CUDA runtime API names to HIP equivalents so
+// the same C++ source compiles under both nvcc and hipcc. Include this instead
+// of <cuda_runtime_api.h> directly when the file needs the runtime API.
+//
+// On NVIDIA platforms the CUDA headers are included as-is and every macro below
+// resolves to the original CUDA symbol, so there is zero overhead.
+//
+// Supported ROCm targets:
+// gfx1100 — RX 7900 XTX / XT
+// gfx1101 — RX 7900 GRE
+// gfx1102 — RX 7700 / XT
+// gfx1103 — RX 7600 / XT
+// gfx1200 — RX 9060 family
+// gfx1201 — RX 9070 family / Radeon AI PRO R9700
+
+#if defined(__HIP_PLATFORM_AMD__) || defined(USE_ROCM)
+
+#define FREETOKEN_USE_ROCM 1
+
+// --- HIP runtime headers ---
+#include <hip/hip_runtime.h>
+#include <hip/hip_runtime_api.h>
+
+// --- API name mapping (CUDA -> HIP) ---
+// HIP already defines most cuda* names as macros that expand to hip* equivalents
+// via hip_runtime.h, but a few are missing or differ in signature. Define them
+// here so call-sites stay unchanged.
+
+#ifndef cudaSuccess
+#define cudaSuccess hipSuccess
+#endif
+
+#ifndef cudaError_t
+#define cudaError_t hipError_t
+#endif
+
+#ifndef cudaGetErrorString
+#define cudaGetErrorString hipGetErrorString
+#endif
+
+#ifndef cudaGetLastError
+#define cudaGetLastError hipGetLastError
+#endif
+
+#ifndef cudaMallocHost
+#define cudaMallocHost hipHostMalloc
+#endif
+
+#ifndef cudaFreeHost
+#define cudaFreeHost hipHostFree
+#endif
+
+#ifndef cudaHostAlloc
+#define cudaHostAlloc hipHostMalloc
+#endif
+
+#ifndef cudaHostRegister
+#define cudaHostRegister hipHostRegister
+#endif
+
+#ifndef cudaHostRegisterPortable
+#define cudaHostRegisterPortable hipHostRegisterPortable
+#endif
+
+#ifndef cudaHostRegisterMapped
+#define cudaHostRegisterMapped hipHostRegisterMapped
+#endif
+
+#ifndef cudaHostAllocPortable
+#define cudaHostAllocPortable hipHostMallocPortable
+#endif
+
+#ifndef cudaHostAllocMapped
+#define cudaHostAllocMapped hipHostMallocMapped
+#endif
+
+#ifndef cudaHostGetDevicePointer
+#define cudaHostGetDevicePointer hipHostGetDevicePointer
+#endif
+
+#ifndef cudaGetDevice
+#define cudaGetDevice hipGetDevice
+#endif
+
+#ifndef cudaDriverGetVersion
+#define cudaDriverGetVersion hipDriverGetVersion
+#endif
+
+#ifndef cudaDeviceGetAttribute
+#define cudaDeviceGetAttribute hipDeviceGetAttribute
+#endif
+
+#ifndef cudaDevAttrUnifiedAddressing
+#define cudaDevAttrUnifiedAddressing hipDeviceAttributeUnifiedAddressing
+#endif
+
+#ifndef cudaDevAttrCanUseHostPointerForRegisteredMem
+// HIP does not expose this attribute; assume UVA identity on ROCm (true on Linux).
+// TODO(ROCm): re-enable proper UVA query if HIP adds this attribute.
+#define cudaDevAttrCanUseHostPointerForRegisteredMem hipDeviceAttributeUnifiedAddressing
+#endif
+
+#ifndef cudaFuncSetAttribute
+#define cudaFuncSetAttribute hipFuncSetAttribute
+#endif
+
+#ifndef cudaFuncAttributeMaxDynamicSharedMemorySize
+#define cudaFuncAttributeMaxDynamicSharedMemorySize hipFuncAttributeMaxDynamicSharedMemorySize
+#endif
+
+#ifndef cudaLaunchKernelEx
+// ROCm 7 exposes the CUDA-compatible extended launch configuration through HIP.
+#define cudaLaunchKernelEx hipLaunchKernelEx
+#endif
+
+#ifndef cudaLaunchConfig_t
+#define cudaLaunchConfig_t hipLaunchConfig_t
+#endif
+
+#ifndef cudaLaunchAttribute
+#define cudaLaunchAttribute hipLaunchAttribute
+#endif
+
+#ifndef cudaLaunchAttributeProgrammaticStreamSerialization
+// PDL (Programmatic Dependent Launch) is NVIDIA-specific.
+// TODO(ROCm): PDL has no ROCm equivalent — disabled, may affect overlap scheduling latency.
+#define cudaLaunchAttributeProgrammaticStreamSerialization 0
+#endif
+
+#ifndef cudaStream_t
+#define cudaStream_t hipStream_t
+#endif
+
+#ifndef cudaStreamSynchronize
+#define cudaStreamSynchronize hipStreamSynchronize
+#endif
+
+#ifndef cudaLaunchHostFunc
+#define cudaLaunchHostFunc hipLaunchHostFunc
+#endif
+
+#ifndef CUDART_CB
+#define CUDART_CB
+#endif
+
+#ifndef __grid_constant__
+#define __grid_constant__
+#endif
+
+#ifndef dim3
+// HIP already provides dim3; this is a no-op guard.
+#endif
+
+#else // NVIDIA CUDA path
+
+#define FREETOKEN_USE_ROCM 0
+
+#include <cuda_runtime_api.h>
+
+#endif
diff --git a/python/freetoken/kernel/csrc/include/freetoken/utils.cuh b/python/freetoken/kernel/csrc/include/freetoken/utils.cuh
index 8e917832..72a49596 100644
--- a/python/freetoken/kernel/csrc/include/freetoken/utils.cuh
+++ b/python/freetoken/kernel/csrc/include/freetoken/utils.cuh
@@ -1,5 +1,6 @@
#pragma once
+#include <freetoken/hip_compat.h>
#include <freetoken/utils.h>
#include <dlpack/dlpack.h>
@@ -115,6 +116,10 @@ public:
}
auto with_attr(bool use_pdl) -> LaunchKernel & {
+#if FREETOKEN_USE_ROCM
+ (void)use_pdl;
+ m_config.numAttrs = 0;
+#else
if (use_pdl) {
m_attr_cache.id = ::cudaLaunchAttributeProgrammaticStreamSerialization;
m_attr_cache.val.programmaticStreamSerializationAllowed = 1;
@@ -123,6 +128,7 @@ public:
} else {
m_config.numAttrs = 0;
}
+#endif
return *this;
}
diff --git a/python/freetoken/kernel/csrc/jit/fast_index_copy.cuh b/python/freetoken/kernel/csrc/jit/fast_index_copy.cuh
index bb83c23e..43f3fc1b 100644
--- a/python/freetoken/kernel/csrc/jit/fast_index_copy.cuh
+++ b/python/freetoken/kernel/csrc/jit/fast_index_copy.cuh
@@ -20,6 +20,13 @@ struct device_vec {
namespace details {
+#if FREETOKEN_USE_ROCM
+// Use native Clang vector types so each __builtin_nontemporal_load/store
+// maps to one vectorized 8B/16B global memory instruction.
+using native_uint2 = uint32_t __attribute__((ext_vector_type(2)));
+using native_uint4 = uint32_t __attribute__((ext_vector_type(4)));
+#endif
+
template <std::size_t kUnit>
inline constexpr auto get_mem_package() {
if constexpr (kUnit == 16) {
@@ -34,40 +41,68 @@ inline constexpr auto get_mem_package() {
}
__always_inline __device__ auto load_nc(const uint1* __restrict__ src) -> uint1 {
+#if FREETOKEN_USE_ROCM
+ return uint1{__builtin_nontemporal_load(&src->x)};
+#else
uint32_t tmp;
asm volatile("ld.global.L1::no_allocate.b32 %0,[%1];" : "=r"(tmp) : "l"(src));
return uint1{tmp};
+#endif
}
__always_inline __device__ auto load_nc(const uint2* __restrict__ src) -> uint2 {
+#if FREETOKEN_USE_ROCM
+ const auto value = __builtin_nontemporal_load(reinterpret_cast<const native_uint2*>(src));
+ return __builtin_bit_cast(uint2, value);
+#else
uint32_t tmp0, tmp1;
asm volatile("ld.global.L1::no_allocate.v2.b32 {%0,%1},[%2];" : "=r"(tmp0), "=r"(tmp1) : "l"(src));
return uint2{tmp0, tmp1};
+#endif
}
__always_inline __device__ auto load_nc(const uint4* __restrict__ src) -> uint4 {
+#if FREETOKEN_USE_ROCM
+ const auto value = __builtin_nontemporal_load(reinterpret_cast<const native_uint4*>(src));
+ return __builtin_bit_cast(uint4, value);
+#else
uint32_t tmp0, tmp1, tmp2, tmp3;
asm volatile("ld.global.L1::no_allocate.v4.b32 {%0,%1,%2,%3},[%4];" : "=r"(tmp0), "=r"(tmp1), "=r"(tmp2), "=r"(tmp3) : "l"(src));
return uint4{tmp0, tmp1, tmp2, tmp3};
+#endif
}
__always_inline __device__ void store_nc(uint1* __restrict__ dst, const uint1& value) {
+#if FREETOKEN_USE_ROCM
+ __builtin_nontemporal_store(value.x, &dst->x);
+#else
uint32_t tmp = value.x;
asm volatile("st.global.wt.b32 [%0],%1;" ::"l"(dst), "r"(tmp));
+#endif
}
__always_inline __device__ void store_nc(uint2* __restrict__ dst, const uint2& value) {
+#if FREETOKEN_USE_ROCM
+ __builtin_nontemporal_store(
+ __builtin_bit_cast(native_uint2, value), reinterpret_cast<native_uint2*>(dst));
+#else
uint32_t tmp0 = value.x;
uint32_t tmp1 = value.y;
asm volatile("st.global.wt.v2.b32 [%0],{%1,%2};" ::"l"(dst), "r"(tmp0), "r"(tmp1));
+#endif
}
__always_inline __device__ void store_nc(uint4* __restrict__ dst, const uint4& value) {
+#if FREETOKEN_USE_ROCM
+ __builtin_nontemporal_store(
+ __builtin_bit_cast(native_uint4, value), reinterpret_cast<native_uint4*>(dst));
+#else
uint32_t tmp0 = value.x;
uint32_t tmp1 = value.y;
uint32_t tmp2 = value.z;
uint32_t tmp3 = value.w;
asm volatile("st.global.wt.v4.b32 [%0],{%1,%2,%3,%4};" ::"l"(dst), "r"(tmp0), "r"(tmp1), "r"(tmp2), "r"(tmp3));
+#endif
}
__always_inline __device__ void wait_flag_clear(const int32_t* __restrict__ flag_ptr) {
@@ -75,7 +110,7 @@ __always_inline __device__ void wait_flag_clear(const int32_t* __restrict__ flag
auto* flag = reinterpret_cast<int*>(const_cast<int32_t*>(flag_ptr));
uint32_t sleep_ns = 128;
while (atomicAdd(flag, 0) > 0) {
-#if __CUDA_ARCH__ >= 700
+#if !FREETOKEN_USE_ROCM && __CUDA_ARCH__ >= 700
__nanosleep(sleep_ns);
#endif
sleep_ns = sleep_ns < 2048 ? (sleep_ns << 1) : 2048;
@@ -147,7 +182,7 @@ inline bool host_ptr_identity() {
}
inline void* device_alias(void* ptr, DLDevice dev) {
- if (dev.device_type == kDLCUDA || host_ptr_identity()) {
+ if (dev.device_type == kDLCUDA || dev.device_type == kDLROCM || host_ptr_identity()) {
return ptr;
}
void* mapped = nullptr;
@@ -269,7 +304,7 @@ inline auto get_sync_flag_ptr(
auto flag_dtype = host::SymbolicDType{};
host::TensorMatcher({1})
.with_dtype<int32_t>(flag_dtype)
- .with_device<kDLCUDA>(device)
+ .with_device<kDLCUDA, kDLROCM>(device)
.verify(sync_flag);
return static_cast<int32_t*>(sync_flag.data_ptr());
}
@@ -344,17 +379,17 @@ struct FastIndexCopyKernel {
TensorMatcher({-1, D})
.with_dtype(data_dtype)
- .with_device<kDLCUDA, kDLCUDAHost, kDLCPU>()
+ .with_device<kDLCUDA, kDLROCM, kDLCUDAHost, kDLROCMHost, kDLCPU>()
.verify(src);
TensorMatcher({-1, D})
.with_dtype(data_dtype)
- .with_device<kDLCUDA, kDLCUDAHost, kDLCPU>()
+ .with_device<kDLCUDA, kDLROCM, kDLCUDAHost, kDLROCMHost, kDLCPU>()
.verify(dst);
TensorMatcher({L})
.with_dtype<int32_t, int64_t>(indices_dtype)
- .with_device<kDLCUDA>(device)
+ .with_device<kDLCUDA, kDLROCM>(device)
.verify(src_indices)
.verify(dst_indices);
@@ -363,7 +398,7 @@ struct FastIndexCopyKernel {
const auto num_indices_tensor = num_indices.value();
TensorMatcher({1})
.with_dtype<int64_t>(num_indices_dtype)
- .with_device<kDLCUDA>(device)
+ .with_device<kDLCUDA, kDLROCM>(device)
.verify(num_indices_tensor);
num_indices_data_ptr = static_cast<const int64_t*>(num_indices_tensor.data_ptr());
@@ -529,14 +564,14 @@ struct MultiIndexCopyKernel {
auto indices_dtype = SymbolicDType{};
auto num_indices_dtype = SymbolicDType{};
- TensorMatcher({B}).with_dtype<int64_t>(ptr_dtype).with_device<kDLCUDA>(device)
+ TensorMatcher({B}).with_dtype<int64_t>(ptr_dtype).with_device<kDLCUDA, kDLROCM>(device)
.verify(dst_ptrs).verify(src_ptrs).verify(feat_bytes);
- TensorMatcher({L}).with_dtype<int32_t, int64_t>(indices_dtype).with_device<kDLCUDA>(device)
+ TensorMatcher({L}).with_dtype<int32_t, int64_t>(indices_dtype).with_device<kDLCUDA, kDLROCM>(device)
.verify(dst_indices).verify(src_indices);
const int64_t* valid_length = nullptr;
if (num_indices.has_value()) {
- TensorMatcher({1}).with_dtype<int64_t>(num_indices_dtype).with_device<kDLCUDA>(device)
+ TensorMatcher({1}).with_dtype<int64_t>(num_indices_dtype).with_device<kDLCUDA, kDLROCM>(device)
.verify(num_indices.value());
valid_length = static_cast<const int64_t*>(num_indices.value().data_ptr());
}
diff --git a/python/freetoken/kernel/csrc/pinned_tensor.cpp b/python/freetoken/kernel/csrc/pinned_tensor.cpp
index c3947adf..9355f57a 100644
--- a/python/freetoken/kernel/csrc/pinned_tensor.cpp
+++ b/python/freetoken/kernel/csrc/pinned_tensor.cpp
@@ -1,5 +1,5 @@
#include <cstdint>
-#include <cuda_runtime_api.h>
+#include <freetoken/hip_compat.h>
#include <torch/extension.h>
namespace {
diff --git a/python/freetoken/kernel/pynccl.py b/python/freetoken/kernel/pynccl.py
index 23ea5735..71bde734 100644
--- a/python/freetoken/kernel/pynccl.py
+++ b/python/freetoken/kernel/pynccl.py
@@ -27,6 +27,7 @@ else:
@functools.cache
def _load_nccl_module() -> Module:
+ # TODO(ROCm): NCCL -> RCCL migration for multi-GPU tensor parallelism on AMD.
return load_aot("pynccl", cuda_files=["pynccl.cu"], extra_ldflags=["-lnccl"])
diff --git a/python/freetoken/kernel/triton/activation.py b/python/freetoken/kernel/triton/activation.py
index 0e4d28c8..9645960d 100644
--- a/python/freetoken/kernel/triton/activation.py
+++ b/python/freetoken/kernel/triton/activation.py
@@ -20,8 +20,9 @@ import triton
import triton.language as tl
from triton.language.extra import libdevice
from triton.language.extra.cuda import gdc_wait, gdc_launch_dependents
+from triton.language import target_info
-from freetoken.utils.arch import is_sm90_supported
+from freetoken.utils.arch import is_rocm, is_sm90_supported
SILU = 0
GELU = 1
@@ -50,6 +51,8 @@ def _pdl_supported() -> bool:
@triton.jit
def _fast_tanh(x):
+ if target_info.is_hip():
+ return libdevice.tanh(x)
# PTX tanh.approx.f32 — single HW op, matches flashinfer math::tanh.
return tl.inline_asm_elementwise(
"tanh.approx.f32 $0, $1;", "=f,f", [x],
@@ -59,6 +62,8 @@ def _fast_tanh(x):
@triton.jit
def _fast_ex2(x):
+ if target_info.is_hip():
+ return libdevice.exp2(x)
# PTX ex2.approx.f32 — matches __expf fast path used by flashinfer silu.
return tl.inline_asm_elementwise(
"ex2.approx.f32 $0, $1;", "=f,f", [x],
@@ -141,8 +146,9 @@ def _act_and_mul(
block_d = min(triton.next_power_of_2(d), 1024 if M >= 4096 else 512)
num_stages = 2 if block_d == 1024 else 3
_act_and_mul_kernel[grid](
- o2, x2, d, alpha, limit, ACT=kind, ENABLE_PDL=pdl, launch_pdl=pdl,
+ o2, x2, d, alpha, limit, ACT=kind, ENABLE_PDL=pdl,
BLOCK_D=block_d, num_warps=4, num_stages=num_stages,
+ **({} if is_rocm() else {"launch_pdl": pdl}),
)
return out
diff --git a/python/freetoken/kernel/triton/e4m3_compat.py b/python/freetoken/kernel/triton/e4m3_compat.py
index 1d9f744c..6ea0c753 100644
--- a/python/freetoken/kernel/triton/e4m3_compat.py
+++ b/python/freetoken/kernel/triton/e4m3_compat.py
@@ -59,6 +59,10 @@ def e4m3_native() -> bool:
if _native is None:
if FORCE_EMU:
_native = False
+ elif torch.version.hip is not None:
+ # ROCm reports gfx1101 as capability (11, 0), which is not a CUDA
+ # compute capability and must not select the native fp8e4nv path.
+ _native = False
else:
from freetoken.gpu_select import assigned_visible_gpu
diff --git a/python/freetoken/kernel/triton/norm.py b/python/freetoken/kernel/triton/norm.py
index 3f95c29f..9071e1df 100644
--- a/python/freetoken/kernel/triton/norm.py
+++ b/python/freetoken/kernel/triton/norm.py
@@ -30,7 +30,7 @@ import triton
import triton.language as tl
from triton.language.extra.cuda import gdc_launch_dependents, gdc_wait
-from freetoken.utils.arch import is_sm90_supported
+from freetoken.utils.arch import is_rocm, is_sm90_supported
_HEUR = {"BLOCK": lambda a: triton.next_power_of_2(a["H"])}
@@ -144,7 +144,8 @@ def _rmsnorm(input, weight, eps, out, gemma: bool):
pdl = contig and is_sm90_supported()
_rmsnorm_kernel[(A, B)](
out, input, weight, eps, H, sxa, sxb, soa, sob,
- CONTIG=contig, ENABLE_PDL=pdl, launch_pdl=pdl, GEMMA=gemma,
+ CONTIG=contig, ENABLE_PDL=pdl, GEMMA=gemma,
+ **({} if is_rocm() else {"launch_pdl": pdl}),
num_warps=_num_warps(A * B), num_stages=1,
)
return out
@@ -172,7 +173,8 @@ def _fused_add_rmsnorm(input, residual, weight, eps, gemma: bool):
pdl = contig and is_sm90_supported()
_fused_add_rmsnorm_kernel[(A, B)](
input, residual, weight, eps, H, sxa, sxb, sra, srb,
- CONTIG=contig, ENABLE_PDL=pdl, launch_pdl=pdl, GEMMA=gemma,
+ CONTIG=contig, ENABLE_PDL=pdl, GEMMA=gemma,
+ **({} if is_rocm() else {"launch_pdl": pdl}),
num_warps=_num_warps(A * B), num_stages=1,
)
diff --git a/python/freetoken/kernel/utils.py b/python/freetoken/kernel/utils.py
index cdd765c7..9af61526 100644
--- a/python/freetoken/kernel/utils.py
+++ b/python/freetoken/kernel/utils.py
@@ -5,6 +5,7 @@ import importlib
import os
import pathlib
import re
+from functools import cache
from typing import TYPE_CHECKING, Iterator, List, NamedTuple, Tuple, TypeAlias, Union
if TYPE_CHECKING:
@@ -20,15 +21,23 @@ _TRUE_VALUES = {"1", "true", "yes", "on"}
DEFAULT_INCLUDE = [str(KERNEL_PATH / "include")]
DEFAULT_CFLAGS = ["-std=c++20", "-O3"]
DEFAULT_CUDA_CFLAGS = ["-std=c++20", "-O3", "--expt-relaxed-constexpr"]
+DEFAULT_HIP_CFLAGS = ["-std=c++20", "-O3"]
DEFAULT_LDFLAGS = []
-ARCH_LIST_ENV = "TVM_FFI_CUDA_ARCH_LIST"
+CUDA_ARCH_LIST_ENV = "TVM_FFI_CUDA_ARCH_LIST"
+ROCM_ARCH_LIST_ENV = "TVM_FFI_ROCM_ARCH_LIST"
+
+
+def _is_rocm() -> bool:
+ import torch
+
+ return getattr(torch.version, "hip", None) is not None
def _cuda_arch_list() -> List[str]:
"""Archs a CUDA build targets: the AOT build's TVM_FFI_CUDA_ARCH_LIST, else the GPU this process is bound to."""
- arch_list = os.getenv(ARCH_LIST_ENV, "").split()
+ arch_list = os.getenv(CUDA_ARCH_LIST_ENV, "").split()
if arch_list:
return arch_list
import torch
@@ -39,17 +48,34 @@ def _cuda_arch_list() -> List[str]:
return [f"{major}.{minor}"]
+def _rocm_arch_list() -> List[str]:
+ """The single ROCm GPU this process is bound to, or an explicit cross-compile target."""
+ from freetoken.utils.arch import get_rocm_gfx_arch
+
+ arch = get_rocm_gfx_arch()
+ if arch is None:
+ raise RuntimeError(
+ "Could not determine the bound ROCm GPU architecture; select a visible GPU "
+ "or set FREETOKEN_ROCM_ARCH for cross-compilation"
+ )
+ return [arch]
+
+
@contextlib.contextmanager
-def _pin_tvm_ffi_arch_ctx(arch_list: List[str]) -> Iterator[None]:
- """Hand tvm-ffi the arch list through its env var for one build. Left unset, tvm-ffi asks nvidia-smi and takes the first GPU listed, which under --gpu or CUDA_VISIBLE_DEVICES on a mixed box is not the bound one."""
- if os.getenv(ARCH_LIST_ENV) or not arch_list:
+def _pin_tvm_ffi_arch_ctx(arch_list: List[str], env_var: str) -> Iterator[None]:
+ """Pin tvm-ffi to the resolved targets for one build, then restore the caller's environment."""
+ if not arch_list:
yield
return
- os.environ[ARCH_LIST_ENV] = " ".join(arch_list)
+ old_arch_list = os.environ.get(env_var)
+ os.environ[env_var] = " ".join(arch_list)
try:
yield
finally:
- os.environ.pop(ARCH_LIST_ENV, None)
+ if old_arch_list is None:
+ os.environ.pop(env_var, None)
+ else:
+ os.environ[env_var] = old_arch_list
def _cuda_cflags(extra: List[str], arch_list: List[str]) -> List[str]:
@@ -63,6 +89,61 @@ def _cuda_cflags(extra: List[str], arch_list: List[str]) -> List[str]:
cc = max(arch_list, key=_rank).rstrip("a").replace(".", "")
flags = flags + [f"-gencode=arch=compute_{cc},code=compute_{cc}"]
return flags
+
+
+def _hip_cflags(extra: List[str], arch_list: List[str]) -> List[str]:
+ """HIP flags for a kernel build on ROCm."""
+ # TODO(ROCm): Triton autotune configs need RDNA-specific tuning (wave count, LDS size).
+ return DEFAULT_HIP_CFLAGS + extra + [f"--offload-arch={arch}" for arch in arch_list]
+
+
+@cache
+def _rocm_link_flags() -> List[str]:
+ """Make ROCm's runtime library discoverable to JIT link commands.
+
+ Traditional ROCm installs provide ``libamdhip64.so`` under ``$ROCM_HOME/lib``.
+ ROCm 7.14 Python SDK images only provide the versioned soname, while TVM-FFI
+ still links with ``-lamdhip64``. Supply a cache-local unversioned symlink via
+ an explicit linker search path without modifying the Python environment.
+ """
+ candidates: list[pathlib.Path] = []
+ if os.getenv("ROCM_HOME"):
+ candidates.append(pathlib.Path(os.environ["ROCM_HOME"]))
+ try:
+ from torch.utils.cpp_extension import ROCM_HOME
+
+ if ROCM_HOME:
+ candidates.append(pathlib.Path(ROCM_HOME))
+ except ImportError:
+ pass
+ spec = importlib.util.find_spec("_rocm_sdk_core")
+ if spec and spec.submodule_search_locations:
+ candidates.append(pathlib.Path(next(iter(spec.submodule_search_locations))))
+ candidates.append(pathlib.Path("/opt/rocm"))
+
+ for rocm_home in dict.fromkeys(candidates):
+ library_dir = rocm_home / "lib"
+ unversioned = library_dir / "libamdhip64.so"
+ link_dir = library_dir
+ if not unversioned.exists():
+ versioned = sorted(library_dir.glob("libamdhip64.so.*"))
+ if not versioned:
+ continue
+ link_dir = pathlib.Path.home() / ".cache" / "freetoken" / "rocm-lib"
+ link_dir.mkdir(parents=True, exist_ok=True)
+ compat_link = link_dir / "libamdhip64.so"
+ if not compat_link.exists() and not compat_link.is_symlink():
+ try:
+ compat_link.symlink_to(versioned[-1])
+ except FileExistsError:
+ # Multiple tensor-parallel ranks may prepare the same cache.
+ pass
+
+ return [f"-L{link_dir}", f"-Wl,-rpath,{library_dir}"]
+
+ raise RuntimeError("Unable to locate libamdhip64 for ROCm JIT linking")
+
+
CPP_TEMPLATE_TYPE: TypeAlias = Union[int, float, bool]
@@ -223,12 +304,16 @@ def load_aot(
if prebuilt is not None:
return prebuilt
+ is_rocm = _is_rocm()
arch_list: List[str] = []
if cuda_files:
- from freetoken.kernel._toolchain import check_nvcc_matches_torch
+ if is_rocm:
+ arch_list = _rocm_arch_list()
+ else:
+ from freetoken.kernel._toolchain import check_nvcc_matches_torch
- check_nvcc_matches_torch()
- arch_list = _cuda_arch_list()
+ check_nvcc_matches_torch()
+ arch_list = _cuda_arch_list()
from tvm_ffi.cpp import load
@@ -242,14 +327,23 @@ def load_aot(
cpp_files = [str((KERNEL_PATH / "src" / f).resolve()) for f in cpp_files]
cuda_files = [str((KERNEL_PATH / "src" / f).resolve()) for f in cuda_files]
- with _pin_tvm_ffi_arch_ctx(arch_list):
+ if is_rocm:
+ cuda_cflags = _hip_cflags(extra_cuda_cflags, arch_list)
+ runtime_ldflags = _rocm_link_flags()
+ arch_list_env = ROCM_ARCH_LIST_ENV
+ else:
+ cuda_cflags = _cuda_cflags(extra_cuda_cflags, arch_list)
+ runtime_ldflags = []
+ arch_list_env = CUDA_ARCH_LIST_ENV
+
+ with _pin_tvm_ffi_arch_ctx(arch_list, arch_list_env):
return load(
name,
cpp_files=cpp_files,
cuda_files=cuda_files,
extra_cflags=DEFAULT_CFLAGS + extra_cflags,
- extra_cuda_cflags=_cuda_cflags(extra_cuda_cflags, arch_list),
- extra_ldflags=DEFAULT_LDFLAGS + extra_ldflags,
+ extra_cuda_cflags=cuda_cflags,
+ extra_ldflags=DEFAULT_LDFLAGS + runtime_ldflags + extra_ldflags,
extra_include_paths=DEFAULT_INCLUDE + extra_include_paths,
build_directory=build_directory,
)
@@ -272,12 +366,16 @@ def load_jit(
if prebuilt is not None:
return prebuilt
+ is_rocm = _is_rocm()
arch_list: List[str] = []
if cuda_files or cuda_wrappers:
- from freetoken.kernel._toolchain import check_nvcc_matches_torch
+ if is_rocm:
+ arch_list = _rocm_arch_list()
+ else:
+ from freetoken.kernel._toolchain import check_nvcc_matches_torch
- check_nvcc_matches_torch()
- arch_list = _cuda_arch_list()
+ check_nvcc_matches_torch()
+ arch_list = _cuda_arch_list()
from tvm_ffi.cpp import load_inline
@@ -300,14 +398,23 @@ def load_jit(
cuda_sources = [f'#include "{path}"' for path in cuda_paths]
cuda_sources += [_make_wrapper(tup) for tup in cuda_wrappers]
- with _pin_tvm_ffi_arch_ctx(arch_list):
+ if is_rocm:
+ cuda_cflags = _hip_cflags(extra_cuda_cflags, arch_list)
+ runtime_ldflags = _rocm_link_flags()
+ arch_list_env = ROCM_ARCH_LIST_ENV
+ else:
+ cuda_cflags = _cuda_cflags(extra_cuda_cflags, arch_list)
+ runtime_ldflags = []
+ arch_list_env = CUDA_ARCH_LIST_ENV
+
+ with _pin_tvm_ffi_arch_ctx(arch_list, arch_list_env):
return load_inline(
name,
cpp_sources=cpp_sources,
cuda_sources=cuda_sources,
extra_cflags=DEFAULT_CFLAGS + extra_cflags,
- extra_cuda_cflags=_cuda_cflags(extra_cuda_cflags, arch_list),
- extra_ldflags=DEFAULT_LDFLAGS + extra_ldflags,
+ extra_cuda_cflags=cuda_cflags,
+ extra_ldflags=DEFAULT_LDFLAGS + runtime_ldflags + extra_ldflags,
extra_include_paths=DEFAULT_INCLUDE + extra_include_paths,
build_directory=build_directory,
)
diff --git a/python/freetoken/utils/__init__.py b/python/freetoken/utils/__init__.py
index 2e4ad15f..af68bf58 100644
--- a/python/freetoken/utils/__init__.py
+++ b/python/freetoken/utils/__init__.py
@@ -1,5 +1,7 @@
from .arch import (
is_arch_supported,
+ is_rocm,
+ get_rocm_gfx_arch,
is_sm90_family,
is_sm90_supported,
is_sm100_family,
@@ -35,6 +37,8 @@ __all__ = [
"load_toolcall_anchor_id",
"init_logger",
"is_arch_supported",
+ "is_rocm",
+ "get_rocm_gfx_arch",
"is_sm90_family",
"is_sm90_supported",
"is_sm100_family",
diff --git a/python/freetoken/utils/arch.py b/python/freetoken/utils/arch.py
index 8c1c6c3d..30450596 100644
--- a/python/freetoken/utils/arch.py
+++ b/python/freetoken/utils/arch.py
@@ -1,14 +1,66 @@
from __future__ import annotations
import functools
+import os
+import re
from typing import Tuple
+_GFX_ARCH_RE = re.compile(r"gfx\d+[a-z]?")
+
+
+def _gfx_arch_from(value: object) -> str | None:
+ match = _GFX_ARCH_RE.search(str(value).lower())
+ return match.group(0) if match else None
+
+
+def is_rocm() -> bool:
+ """True when torch is built for ROCm (AMD GPU) instead of CUDA."""
+ import torch
+ return getattr(torch.version, "hip", None) is not None
+
+
+def get_rocm_gfx_arch() -> str | None:
+ """Return the current AMD GPU target (for example ``gfx1201``).
+
+ Prefer the runtime device because build variables may contain multiple
+ semicolon-separated targets. Environment variables remain useful for
+ cross-compilation and systems where no GPU is currently visible; in that
+ fallback mode the first target is returned. The result is process-cached
+ for FreeToken's one-process-per-GPU execution model, so callers must select
+ the intended device before the first call.
+ """
+ if not is_rocm():
+ return None
+
+ import torch
+
+ if torch.cuda.is_available():
+ try:
+ props = torch.cuda.get_device_properties(torch.cuda.current_device())
+ for attr in ("gcnArchName", "arch"):
+ arch = _gfx_arch_from(getattr(props, attr, ""))
+ if arch:
+ return arch
+ except (AttributeError, RuntimeError):
+ pass
+
+ for env_var in ("FREETOKEN_ROCM_ARCH", "PYTORCH_ROCM_ARCH", "HCC_AMDGPU_TARGET"):
+ arch = _gfx_arch_from(os.getenv(env_var, ""))
+ if arch:
+ return arch
+ return None
+
+
@functools.cache
def _get_torch_cuda_version() -> Tuple[int, int] | None:
import torch
import torch.version
+ if is_rocm():
+ return None
if not torch.cuda.is_available() or not torch.version.cuda:
return None
return torch.cuda.get_device_capability()
diff --git a/setup.py b/setup.py
index 7132eea0..997745f1 100644
--- a/setup.py
+++ b/setup.py
@@ -1,15 +1,17 @@
from __future__ import annotations
import importlib.util
+import os
from pathlib import Path
import sys
from setuptools import setup
-from torch.utils.cpp_extension import BuildExtension, CUDA_HOME, CppExtension
+from torch.utils.cpp_extension import BuildExtension, CUDA_HOME, CppExtension, ROCM_HOME
ROOT = Path(__file__).parent
+KERNEL_INCLUDE = str(ROOT / "python" / "freetoken" / "kernel" / "csrc" / "include")
def _check_toolchain() -> None:
@@ -20,6 +22,43 @@ def _check_toolchain() -> None:
module.check_nvcc_matches_torch()
+def _is_rocm() -> bool:
+ import torch
+ return getattr(torch.version, "hip", None) is not None
+
+
+def _rocm_paths() -> tuple[list[str], list[str], str]:
+ candidates: list[Path] = []
+ if os.getenv("ROCM_HOME"):
+ candidates.append(Path(os.environ["ROCM_HOME"]))
+ if ROCM_HOME:
+ candidates.append(Path(ROCM_HOME))
+
+ # ROCm 7.14 PyTorch images ship the SDK as a Python package instead of
+ # installing it at /opt/rocm.
+ spec = importlib.util.find_spec("_rocm_sdk_core")
+ if spec and spec.submodule_search_locations:
+ candidates.append(Path(next(iter(spec.submodule_search_locations))))
+ candidates.append(Path("/opt/rocm"))
+
+ for rocm_home in dict.fromkeys(candidates):
+ include_dir = rocm_home / "include"
+ library_dir = rocm_home / "lib"
+ if not (include_dir / "hip" / "hip_runtime.h").exists():
+ continue
+ if (library_dir / "libamdhip64.so").exists():
+ return [str(include_dir)], [str(library_dir)], "amdhip64"
+ versioned = sorted(library_dir.glob("libamdhip64.so.*"))
+ if versioned:
+ return [str(include_dir)], [str(library_dir)], f":{versioned[-1].name}"
+
+ searched = ", ".join(str(path) for path in dict.fromkeys(candidates))
+ raise RuntimeError(
+ "A ROCm SDK with HIP headers and libamdhip64 is required to build on ROCm; "
+ f"searched: {searched}. Set ROCM_HOME to override."
+ )
+
+
def _cuda_runtime_paths() -> tuple[list[str], list[str]]:
if CUDA_HOME is None:
raise RuntimeError(
@@ -33,7 +72,21 @@ def _cuda_runtime_paths() -> tuple[list[str], list[str]]:
return [str(cuda_home / "include")], library_dirs
-cuda_include_dirs, cuda_library_dirs = _cuda_runtime_paths()
+IS_ROCM = _is_rocm()
+
+if IS_ROCM:
+ runtime_include_dirs, runtime_library_dirs, runtime_lib = _rocm_paths()
+ runtime_link_args = [f"-Wl,-rpath,{runtime_library_dirs[0]}"]
+ # These extensions contain host code only. BuildExtension supplies the ROCm
+ # platform defines to the C++ compiler; offload architecture flags belong on
+ # HIP device sources and would be rejected by the host compiler here.
+ extra_compile = ["-O3", "-std=c++17"]
+else:
+ runtime_include_dirs, runtime_library_dirs = _cuda_runtime_paths()
+ runtime_lib = "cudart"
+ runtime_link_args = []
+ extra_compile = ["-O3", "-std=c++17"]
+
_check_toolchain()
@@ -44,12 +97,13 @@ setup(
sources=[
"python/freetoken/kernel/csrc/pinned_tensor.cpp",
],
- include_dirs=cuda_include_dirs,
- library_dirs=cuda_library_dirs,
- libraries=["cudart"],
- extra_compile_args=["-O3", "-std=c++17"],
+ include_dirs=[KERNEL_INCLUDE, *runtime_include_dirs],
+ library_dirs=runtime_library_dirs,
+ libraries=[runtime_lib],
+ extra_compile_args=extra_compile,
+ extra_link_args=runtime_link_args,
),
- # CPU-compute MoE executor for --moe-backend cpu. Links cudart for the
+ # CPU-compute MoE executor for --moe-backend cpu. Links cudart/amdhip64 for the
# cudaLaunchHostFunc submit/sync graph nodes; the bf16 GEMV microkernels
# use per-function target attributes (avx512bf16/avx512f) + a runtime
# __builtin_cpu_supports dispatch, so the single binary stays portable
@@ -59,10 +113,11 @@ setup(
sources=[
"python/freetoken/kernel/csrc/cpu_moe/cpu_moe_ext.cpp",
],
- include_dirs=cuda_include_dirs,
- library_dirs=cuda_library_dirs,
- libraries=["cudart"],
- extra_compile_args=["-O3", "-std=c++17", "-pthread"],
+ include_dirs=[KERNEL_INCLUDE, *runtime_include_dirs],
+ library_dirs=runtime_library_dirs,
+ libraries=[runtime_lib],
+ extra_compile_args=extra_compile + ["-pthread"],
+ extra_link_args=runtime_link_args,
),
# disk-backed row store (PLE / Engram tables); Linux-only until the TableFile/BatchReader seams grow Windows bodies
*([
diff --git a/tests/kernels/test_e4m3_compat.py b/tests/kernels/test_e4m3_compat.py
index 8bcb8e58..fa8a1f1d 100644
--- a/tests/kernels/test_e4m3_compat.py
+++ b/tests/kernels/test_e4m3_compat.py
@@ -41,6 +41,29 @@ def _native_cc() -> bool:
return torch.cuda.get_device_capability() >= (8, 9)
[email protected](torch.version.hip is None, reason="needs ROCm")
+def test_rocm_host_and_compile_time_native_gates_match():
+ """The host wrapper and Triton constexpr must choose the same buffer ABI."""
+ import triton
+ import triton.language as tl
+
+ from freetoken.kernel.triton.e4m3_compat import e4m3_native, e4m3_native_cx
+
+ @triton.jit
+ def gate_kernel(output):
+ if e4m3_native_cx():
+ value = 1
+ else:
+ value = 0
+ tl.store(output, value)
+
+ output = torch.empty(1, dtype=torch.int32, device="cuda")
+ gate_kernel[(1,)](output)
+
+ assert e4m3_native() is False
+ assert output.item() == int(e4m3_native())
+
+
# ======================================================================================
# 1. Primitives vs the native fp8 unit (needs sm_89+ hardware for the reference).
# ======================================================================================
diff --git a/tests/kernels/test_pinned_tensor.py b/tests/kernels/test_pinned_tensor.py
index e61108fd..2fee4f25 100644
--- a/tests/kernels/test_pinned_tensor.py
+++ b/tests/kernels/test_pinned_tensor.py
@@ -122,7 +122,12 @@ def test_host_device_ptr_is_identity_under_uva():
pytest.skip("non-UVA platform: host_device_ptr rejects unregistered memory instead")
# Under UVA cudaHostGetDevicePointer degenerates to identity for any host pointer
# (no registration validation); rejection of pageable memory only exists on
- # non-identity platforms (Windows/WDDM), where the translation is real.
+ # non-identity CUDA platforms (Windows/WDDM), where the translation is real.
+ # HIP validates registration even though registered/pinned memory uses the
+ # identity address on Linux. Calling it with pageable memory also leaves a
+ # sticky HIP error, so the pinned identity case above is the relevant check.
+ if torch.version.hip is not None:
+ return
pageable = torch.empty(64, dtype=torch.uint8)
ext = _load_pinned_extension()
assert ext.host_device_ptr(pageable.data_ptr()) == pageable.data_ptr()
diff --git a/tests/kernels/test_rocm_launch_kwargs.py b/tests/kernels/test_rocm_launch_kwargs.py
new file mode 100644
index 00000000..52fac545
--- /dev/null
+++ b/tests/kernels/test_rocm_launch_kwargs.py
@@ -0,0 +1,51 @@
+import pytest
+import torch
+
+
+pytestmark = pytest.mark.skipif(
+ not torch.cuda.is_available() or torch.version.hip is None,
+ reason="a ROCm GPU is required",
+)
+
+
+class _KernelRecorder:
+ def __init__(self):
+ self.kwargs = None
+
+ def __getitem__(self, _grid):
+ def launch(*_args, **kwargs):
+ self.kwargs = kwargs
+
+ return launch
+
+
+def test_rocm_activation_launch_omits_cuda_pdl_attribute(monkeypatch):
+ import freetoken.kernel.triton.activation as activation
+
+ recorder = _KernelRecorder()
+ monkeypatch.setattr(activation, "_act_and_mul_kernel", recorder)
+ monkeypatch.setattr(activation, "_pdl_supported", lambda: True)
+ monkeypatch.setattr(activation, "is_rocm", lambda: True)
+
+ activation.silu_and_mul(torch.ones((2, 128), device="cuda"))
+
+ assert recorder.kwargs is not None
+ assert recorder.kwargs["ENABLE_PDL"] is True
+ assert "launch_pdl" not in recorder.kwargs
+
+
+def test_rocm_norm_launch_omits_cuda_pdl_attribute(monkeypatch):
+ import freetoken.kernel.triton.norm as norm
+
+ recorder = _KernelRecorder()
+ monkeypatch.setattr(norm, "_rmsnorm_kernel", recorder)
+ monkeypatch.setattr(norm, "is_sm90_supported", lambda: True)
+ monkeypatch.setattr(norm, "is_rocm", lambda: True)
+
+ x = torch.ones((2, 64), device="cuda")
+ weight = torch.ones(64, device="cuda")
+ norm.rmsnorm(x, weight)
+
+ assert recorder.kwargs is not None
+ assert recorder.kwargs["ENABLE_PDL"] is True
+ assert "launch_pdl" not in recorder.kwargs
diff --git a/tests/utils/test_rocm_arch.py b/tests/utils/test_rocm_arch.py
new file mode 100644
index 00000000..fcfab758
--- /dev/null
+++ b/tests/utils/test_rocm_arch.py
@@ -0,0 +1,127 @@
+import importlib
+import os
+import pathlib
+import sys
+import types
+from types import SimpleNamespace
+
+import torch
+
+from freetoken.utils import arch
+
+
+def _clear_arch_caches() -> None:
+ arch.get_rocm_gfx_arch.cache_clear()
+
+
+def test_rocm_arch_prefers_visible_device_over_multi_arch_build_env(monkeypatch):
+ monkeypatch.setattr(arch, "is_rocm", lambda: True)
+ monkeypatch.setattr(torch.cuda, "is_available", lambda: True)
+ monkeypatch.setattr(torch.cuda, "current_device", lambda: 0)
+ monkeypatch.setattr(
+ torch.cuda,
+ "get_device_properties",
+ lambda _device: SimpleNamespace(gcnArchName="gfx1201:sramecc-:xnack-"),
+ )
+ monkeypatch.setenv("FREETOKEN_ROCM_ARCH", "gfx1100;gfx1200")
+ _clear_arch_caches()
+
+ assert arch.get_rocm_gfx_arch() == "gfx1201"
+
+ _clear_arch_caches()
+
+
+def test_rocm_arch_falls_back_to_cross_compile_env(monkeypatch):
+ monkeypatch.setattr(arch, "is_rocm", lambda: True)
+ monkeypatch.setattr(torch.cuda, "is_available", lambda: False)
+ monkeypatch.setenv("FREETOKEN_ROCM_ARCH", "gfx1200;gfx1201")
+ _clear_arch_caches()
+
+ assert arch.get_rocm_gfx_arch() == "gfx1200"
+
+ _clear_arch_caches()
+
+
+def test_hip_cflags_target_only_resolved_arch(monkeypatch):
+ from freetoken.kernel import utils
+
+ monkeypatch.setenv("FREETOKEN_ROCM_ARCH", "gfx1200;gfx1201")
+ monkeypatch.setattr(arch, "get_rocm_gfx_arch", lambda: "gfx1201")
+
+ arch_list = utils._rocm_arch_list()
+ flags = utils._hip_cflags(["-Wno-unused-command-line-argument"], arch_list)
+
+ assert arch_list == ["gfx1201"]
+ assert "--offload-arch=gfx1201" in flags
+ assert "--offload-arch=gfx1200" not in flags
+
+
+def test_rocm_loaders_pin_tvm_ffi_to_resolved_arch(monkeypatch):
+ from freetoken.kernel import utils
+
+ builds = []
+
+ def record_build(*_args, **kwargs):
+ builds.append(
+ (
+ os.environ[utils.ROCM_ARCH_LIST_ENV],
+ kwargs["extra_cuda_cflags"],
+ )
+ )
+ return object()
+
+ tvm_ffi = types.ModuleType("tvm_ffi")
+ tvm_ffi.__path__ = []
+ tvm_ffi_cpp = types.ModuleType("tvm_ffi.cpp")
+ tvm_ffi_cpp.load = record_build
+ tvm_ffi_cpp.load_inline = record_build
+ monkeypatch.setitem(sys.modules, "tvm_ffi", tvm_ffi)
+ monkeypatch.setitem(sys.modules, "tvm_ffi.cpp", tvm_ffi_cpp)
+ monkeypatch.setenv(utils.DISABLE_KERNEL_CACHE_ENV, "1")
+ monkeypatch.setenv(utils.ROCM_ARCH_LIST_ENV, "gfx1100 gfx1200")
+ monkeypatch.setattr(utils, "_is_rocm", lambda: True)
+ monkeypatch.setattr(utils, "_rocm_arch_list", lambda: ["gfx1201"])
+ monkeypatch.setattr(utils, "_rocm_link_flags", lambda: [])
+
+ utils.load_aot("test_rocm_aot_arch", cuda_files=["unused.cu"])
+ utils.load_jit("test_rocm_jit_arch", cuda_files=["unused.cu"])
+
+ assert builds == [
+ ("gfx1201", [*utils.DEFAULT_HIP_CFLAGS, "--offload-arch=gfx1201"]),
+ ("gfx1201", [*utils.DEFAULT_HIP_CFLAGS, "--offload-arch=gfx1201"]),
+ ]
+ assert os.environ[utils.ROCM_ARCH_LIST_ENV] == "gfx1100 gfx1200"
+
+
+def test_rocm_link_flags_support_versioned_modular_sdk(monkeypatch, tmp_path):
+ import torch.utils.cpp_extension as cpp_extension
+
+ from freetoken.kernel import utils
+
+ sdk = tmp_path / "sdk"
+ library_dir = sdk / "lib"
+ library_dir.mkdir(parents=True)
+ versioned_runtime = library_dir / "libamdhip64.so.7"
+ versioned_runtime.write_bytes(b"")
+ real_find_spec = importlib.util.find_spec
+
+ def find_spec(name: str):
+ if name == "_rocm_sdk_core":
+ return SimpleNamespace(submodule_search_locations=[str(sdk)])
+ return real_find_spec(name)
+
+ monkeypatch.delenv("ROCM_HOME", raising=False)
+ monkeypatch.setattr(cpp_extension, "ROCM_HOME", None)
+ monkeypatch.setattr(importlib.util, "find_spec", find_spec)
+ monkeypatch.setattr(pathlib.Path, "home", lambda: tmp_path)
+ utils._rocm_link_flags.cache_clear()
+
+ flags = utils._rocm_link_flags()
+
+ compat_dir = tmp_path / ".cache" / "freetoken" / "rocm-lib"
+ compat_link = compat_dir / "libamdhip64.so"
+ assert f"-L{compat_dir}" in flags
+ assert f"-Wl,-rpath,{library_dir}" in flags
+ assert compat_link.resolve() == versioned_runtime.resolve()
+
+ utils._rocm_link_flags.cache_clear()