FreeToken brings datacenter-scale model serving to your desktop. Run massive models locally, fast and efficiently.
feat(rocm): add RDNA3 and RDNA4 runtime foundation (#132)
ZihaoMu0d652e73a452
diff --git a/docs/install.md b/docs/install.mdindex 9438c21a..f474216b 100644--- a/docs/install.md+++ b/docs/install.md@@ -1,5 +1,7 @@# Install+AMD users: see the [AMD ROCm installation guide (WIP)](install_amd.md).+## Requirements- Linux x86_64, NVIDIA GPU, driver r580+ (CUDA 13)diff --git a/docs/install_amd.md b/docs/install_amd.mdnew file mode 100644index 00000000..d8b0c1bc--- /dev/null+++ b/docs/install_amd.md@@ -0,0 +1,39 @@+# AMD ROCm installation (WIP)++AMD support is experimental and remains a work in progress. See the+[AMD Support roadmap](https://github.com/FlashML-org/FreeToken/issues/541) for+the current integration and qualification status.++## Requirements++- Linux x86_64+- AMD RDNA3/RDNA4 GPU (`gfx1100`-`gfx1103`, `gfx1200`, or `gfx1201`)+- ROCm 7.14+- Python >= 3.10++## Install from source++Use an official ROCm PyTorch image whose PyTorch version satisfies the project's+`torch>=2.11,<2.12` constraint. For RDNA4, the matching ROCm 7.14 image is:++```bash+VIDEO_GID="$(getent group video | cut -d: -f3)"+RENDER_GID="$(getent group render | cut -d: -f3)"+docker run --rm -it \+ --device=/dev/kfd --device=/dev/dri \+ --group-add="$VIDEO_GID" --group-add="$RENDER_GID" --ipc=host \+ --cap-add=SYS_PTRACE --security-opt seccomp=unconfined \+ -e PYTORCH_ROCM_ARCH=gfx1201 -e FREETOKEN_ROCM_ARCH=gfx1201 \+ -v "$PWD:/workspace/FreeToken" -w /workspace/FreeToken \+ rocm/pytorch:rocm7.14_ubuntu24.04_py3.12_pytorch_release_2.11.0 bash+```++Inside the container, preserve the ROCm-enabled PyTorch already supplied by the+image and disable build isolation so it is also used to compile the extensions:++```bash+python -m pip install --no-build-isolation -e .+```++Set both architecture variables to `gfx1200` for RX 9060 family GPUs, or to the+actual target reported by `rocminfo`.diff --git a/pyproject.toml b/pyproject.tomlindex 25cb204b..8d00d2bb 100644--- a/pyproject.toml+++ b/pyproject.toml@@ -23,6 +23,7 @@ classifiers = ["Intended Audience :: Science/Research","Operating System :: POSIX :: Linux","Environment :: GPU :: NVIDIA CUDA",+ "Environment :: GPU :: AMD ROCm","Programming Language :: Python :: 3","Programming Language :: Python :: 3.10","Programming Language :: Python :: 3.11",@@ -59,7 +60,10 @@ dependencies = ["torchvision>=0.26,<0.27","tqdm>=4.66,<5","transformers>=5.16,<5.17",- "triton==3.6.0; platform_system == 'Linux'",+ # CUDA torch 2.11 resolves Triton 3.6; AMD's ROCm 7.14 image supplies its+ # gfx1201-enabled Triton 3.7 build. Keep both supported without replacing the+ # runtime-specific wheel selected by the PyTorch distribution.+ "triton>=3.6,<3.8; platform_system == 'Linux'","uvicorn>=0.30,<1",]diff --git a/python/freetoken/kernel/__main__.py b/python/freetoken/kernel/__main__.pyindex 7be541a6..5b66d484 100644--- a/python/freetoken/kernel/__main__.py+++ b/python/freetoken/kernel/__main__.py@@ -6,27 +6,32 @@ def generate_clangd():import subprocessfrom freetoken.kernel.utils import DEFAULT_INCLUDE- from freetoken.utils import init_logger+ from freetoken.utils import get_rocm_gfx_arch, init_logger, is_rocmfrom tvm_ffi.libinfo import find_dlpack_include_path, find_include_pathlogger = init_logger(__name__)logger.info("Generating .clangd file...")include_paths = [find_include_path(), find_dlpack_include_path()] + DEFAULT_INCLUDE- status = subprocess.run(- args=["nvidia-smi", "--query-gpu=compute_cap", "--format=csv,noheader"],- capture_output=True,- check=True,- )- compute_cap = status.stdout.decode("utf-8").strip().split("\n")[0]- major, minor = compute_cap.split(".")++ # TODO(ROCm): hiprtc JIT cache should be separate from nvcc JIT cache to avoid stale binaries.+ if is_rocm():+ arch_flags = ["-xhip", f"--offload-arch={get_rocm_gfx_arch() or 'gfx1201'}"]+ else:+ try:+ status = subprocess.run(+ args=["nvidia-smi", "--query-gpu=compute_cap", "--format=csv,noheader"],+ capture_output=True,+ check=True,+ )+ compute_cap = status.stdout.decode("utf-8").strip().split("\n")[0]+ major, minor = compute_cap.split(".")+ except (subprocess.CalledProcessError, FileNotFoundError, ValueError):+ import torch++ major, minor = torch.cuda.get_device_capability()+ arch_flags = ["-xcuda", f"--cuda-gpu-arch=sm_{major}{minor}"]compile_flags = ",\n ".join(- [- "-xcuda",- f"--cuda-gpu-arch=sm_{major}{minor}",- "-std=c++20",- "-Wall",- "-Wextra",- ]+ arch_flags + ["-std=c++20", "-Wall", "-Wextra"]+ [f"-isystem{path}" for path in include_paths])clangd_content = f"""diff --git a/python/freetoken/kernel/_toolchain.py b/python/freetoken/kernel/_toolchain.pyindex b49cebbb..93d03221 100644--- a/python/freetoken/kernel/_toolchain.py+++ b/python/freetoken/kernel/_toolchain.py@@ -1,4 +1,4 @@-"""CUDA toolchain/torch consistency checks.+"""CUDA/HIP toolchain/torch consistency checks.Standalone on purpose: setup.py and the kernel-cache build backend load thisfile by path, so it must not import the freetoken package.@@ -16,6 +16,11 @@ ALLOW_MISMATCH_ENV = "FREETOKEN_ALLOW_CUDA_MISMATCH"_TRUE_VALUES = {"1", "true", "yes", "on"}+def _is_rocm() -> bool:+ import torch+ return getattr(torch.version, "hip", None) is not None++def _nvcc_path() -> str | None:from torch.utils.cpp_extension import CUDA_HOME@@ -49,6 +54,8 @@ def check_nvcc_matches_torch() -> None:nvcc-built binaries link libcudart.so.<nvcc major>; at runtime only thetorch wheel's own CUDA runtime is guaranteed to be loadable."""+ if _is_rocm():+ return # ROCm uses hipcc, not nvccif os.getenv(ALLOW_MISMATCH_ENV, "").strip().lower() in _TRUE_VALUES:returntorch_major = torch_cuda_major()diff --git a/python/freetoken/kernel/backend.py b/python/freetoken/kernel/backend.pyindex 2a886cf7..6d1b4a9f 100644--- a/python/freetoken/kernel/backend.py+++ b/python/freetoken/kernel/backend.py@@ -47,6 +47,24 @@ def device_capability() -> tuple[int, int]:return (int(major), int(minor))+def is_rocm() -> bool:+ """True when torch is built for ROCm (AMD GPU)."""+ import torch+ return getattr(torch.version, "hip", None) is not None+++def driver_hip_version() -> int | None:+ """ROCm driver version, or None if undetermined."""+ # TODO(ROCm): flashinfer/sgl_kernel have no ROCm builds — Triton fallback is used.+ try:+ from freetoken.kernel.pinned import _load_pinned_extension+ return int(_load_pinned_extension().driver_cuda_version()) or None+ except Exception:+ return None++@functools.cachedef driver_cuda_version() -> int | None:"""Max CUDA version the installed NVIDIA driver supports (``13000`` == CUDA 13.0),diff --git a/python/freetoken/kernel/csrc/cpu_moe/cpu_moe_ext.cpp b/python/freetoken/kernel/csrc/cpu_moe/cpu_moe_ext.cppindex 8349123b..e2cb0d9c 100644--- a/python/freetoken/kernel/csrc/cpu_moe/cpu_moe_ext.cpp+++ b/python/freetoken/kernel/csrc/cpu_moe/cpu_moe_ext.cpp@@ -29,7 +29,7 @@#include <thread>#include <vector>-#include <cuda_runtime_api.h>+#include <freetoken/hip_compat.h>#include <torch/extension.h>#if defined(__linux__)diff --git a/python/freetoken/kernel/csrc/include/freetoken/hip_compat.h b/python/freetoken/kernel/csrc/include/freetoken/hip_compat.hnew file mode 100644index 00000000..1898b411--- /dev/null+++ b/python/freetoken/kernel/csrc/include/freetoken/hip_compat.h@@ -0,0 +1,162 @@+#pragma once++// HIP compatibility shim: maps CUDA runtime API names to HIP equivalents so+// the same C++ source compiles under both nvcc and hipcc. Include this instead+// of <cuda_runtime_api.h> directly when the file needs the runtime API.+//+// On NVIDIA platforms the CUDA headers are included as-is and every macro below+// resolves to the original CUDA symbol, so there is zero overhead.+//+// Supported ROCm targets:+// gfx1100 — RX 7900 XTX / XT+// gfx1101 — RX 7900 GRE+// gfx1102 — RX 7700 / XT+// gfx1103 — RX 7600 / XT+// gfx1200 — RX 9060 family+// gfx1201 — RX 9070 family / Radeon AI PRO R9700++#if defined(__HIP_PLATFORM_AMD__) || defined(USE_ROCM)++#define FREETOKEN_USE_ROCM 1++// --- HIP runtime headers ---+#include <hip/hip_runtime.h>+#include <hip/hip_runtime_api.h>++// --- API name mapping (CUDA -> HIP) ---+// HIP already defines most cuda* names as macros that expand to hip* equivalents+// via hip_runtime.h, but a few are missing or differ in signature. Define them+// here so call-sites stay unchanged.++#ifndef cudaSuccess+#define cudaSuccess hipSuccess+#endif++#ifndef cudaError_t+#define cudaError_t hipError_t+#endif++#ifndef cudaGetErrorString+#define cudaGetErrorString hipGetErrorString+#endif++#ifndef cudaGetLastError+#define cudaGetLastError hipGetLastError+#endif++#ifndef cudaMallocHost+#define cudaMallocHost hipHostMalloc+#endif++#ifndef cudaFreeHost+#define cudaFreeHost hipHostFree+#endif++#ifndef cudaHostAlloc+#define cudaHostAlloc hipHostMalloc+#endif++#ifndef cudaHostRegister+#define cudaHostRegister hipHostRegister+#endif++#ifndef cudaHostRegisterPortable+#define cudaHostRegisterPortable hipHostRegisterPortable+#endif++#ifndef cudaHostRegisterMapped+#define cudaHostRegisterMapped hipHostRegisterMapped+#endif++#ifndef cudaHostAllocPortable+#define cudaHostAllocPortable hipHostMallocPortable+#endif++#ifndef cudaHostAllocMapped+#define cudaHostAllocMapped hipHostMallocMapped+#endif++#ifndef cudaHostGetDevicePointer+#define cudaHostGetDevicePointer hipHostGetDevicePointer+#endif++#ifndef cudaGetDevice+#define cudaGetDevice hipGetDevice+#endif++#ifndef cudaDriverGetVersion+#define cudaDriverGetVersion hipDriverGetVersion+#endif++#ifndef cudaDeviceGetAttribute+#define cudaDeviceGetAttribute hipDeviceGetAttribute+#endif++#ifndef cudaDevAttrUnifiedAddressing+#define cudaDevAttrUnifiedAddressing hipDeviceAttributeUnifiedAddressing+#endif++#ifndef cudaDevAttrCanUseHostPointerForRegisteredMem+// HIP does not expose this attribute; assume UVA identity on ROCm (true on Linux).+// TODO(ROCm): re-enable proper UVA query if HIP adds this attribute.+#define cudaDevAttrCanUseHostPointerForRegisteredMem hipDeviceAttributeUnifiedAddressing+#endif++#ifndef cudaFuncSetAttribute+#define cudaFuncSetAttribute hipFuncSetAttribute+#endif++#ifndef cudaFuncAttributeMaxDynamicSharedMemorySize+#define cudaFuncAttributeMaxDynamicSharedMemorySize hipFuncAttributeMaxDynamicSharedMemorySize+#endif++#ifndef cudaLaunchKernelEx+// ROCm 7 exposes the CUDA-compatible extended launch configuration through HIP.+#define cudaLaunchKernelEx hipLaunchKernelEx+#endif++#ifndef cudaLaunchConfig_t+#define cudaLaunchConfig_t hipLaunchConfig_t+#endif++#ifndef cudaLaunchAttribute+#define cudaLaunchAttribute hipLaunchAttribute+#endif++#ifndef cudaLaunchAttributeProgrammaticStreamSerialization+// PDL (Programmatic Dependent Launch) is NVIDIA-specific.+// TODO(ROCm): PDL has no ROCm equivalent — disabled, may affect overlap scheduling latency.+#define cudaLaunchAttributeProgrammaticStreamSerialization 0+#endif++#ifndef cudaStream_t+#define cudaStream_t hipStream_t+#endif++#ifndef cudaStreamSynchronize+#define cudaStreamSynchronize hipStreamSynchronize+#endif++#ifndef cudaLaunchHostFunc+#define cudaLaunchHostFunc hipLaunchHostFunc+#endif++#ifndef CUDART_CB+#define CUDART_CB+#endif++#ifndef __grid_constant__+#define __grid_constant__+#endif++#ifndef dim3+// HIP already provides dim3; this is a no-op guard.+#endif++#else // NVIDIA CUDA path++#define FREETOKEN_USE_ROCM 0++#include <cuda_runtime_api.h>++#endifdiff --git a/python/freetoken/kernel/csrc/include/freetoken/utils.cuh b/python/freetoken/kernel/csrc/include/freetoken/utils.cuhindex 8e917832..72a49596 100644--- a/python/freetoken/kernel/csrc/include/freetoken/utils.cuh+++ b/python/freetoken/kernel/csrc/include/freetoken/utils.cuh@@ -1,5 +1,6 @@#pragma once+#include <freetoken/hip_compat.h>#include <freetoken/utils.h>#include <dlpack/dlpack.h>@@ -115,6 +116,10 @@ public:}auto with_attr(bool use_pdl) -> LaunchKernel & {+#if FREETOKEN_USE_ROCM+ (void)use_pdl;+ m_config.numAttrs = 0;+#elseif (use_pdl) {m_attr_cache.id = ::cudaLaunchAttributeProgrammaticStreamSerialization;m_attr_cache.val.programmaticStreamSerializationAllowed = 1;@@ -123,6 +128,7 @@ public:} else {m_config.numAttrs = 0;}+#endifreturn *this;}diff --git a/python/freetoken/kernel/csrc/jit/fast_index_copy.cuh b/python/freetoken/kernel/csrc/jit/fast_index_copy.cuhindex bb83c23e..43f3fc1b 100644--- a/python/freetoken/kernel/csrc/jit/fast_index_copy.cuh+++ b/python/freetoken/kernel/csrc/jit/fast_index_copy.cuh@@ -20,6 +20,13 @@ struct device_vec {namespace details {+#if FREETOKEN_USE_ROCM+// Use native Clang vector types so each __builtin_nontemporal_load/store+// maps to one vectorized 8B/16B global memory instruction.+using native_uint2 = uint32_t __attribute__((ext_vector_type(2)));+using native_uint4 = uint32_t __attribute__((ext_vector_type(4)));+#endif+template <std::size_t kUnit>inline constexpr auto get_mem_package() {if constexpr (kUnit == 16) {@@ -34,40 +41,68 @@ inline constexpr auto get_mem_package() {}__always_inline __device__ auto load_nc(const uint1* __restrict__ src) -> uint1 {+#if FREETOKEN_USE_ROCM+ return uint1{__builtin_nontemporal_load(&src->x)};+#elseuint32_t tmp;asm volatile("ld.global.L1::no_allocate.b32 %0,[%1];" : "=r"(tmp) : "l"(src));return uint1{tmp};+#endif}__always_inline __device__ auto load_nc(const uint2* __restrict__ src) -> uint2 {+#if FREETOKEN_USE_ROCM+ const auto value = __builtin_nontemporal_load(reinterpret_cast<const native_uint2*>(src));+ return __builtin_bit_cast(uint2, value);+#elseuint32_t tmp0, tmp1;asm volatile("ld.global.L1::no_allocate.v2.b32 {%0,%1},[%2];" : "=r"(tmp0), "=r"(tmp1) : "l"(src));return uint2{tmp0, tmp1};+#endif}__always_inline __device__ auto load_nc(const uint4* __restrict__ src) -> uint4 {+#if FREETOKEN_USE_ROCM+ const auto value = __builtin_nontemporal_load(reinterpret_cast<const native_uint4*>(src));+ return __builtin_bit_cast(uint4, value);+#elseuint32_t tmp0, tmp1, tmp2, tmp3;asm volatile("ld.global.L1::no_allocate.v4.b32 {%0,%1,%2,%3},[%4];" : "=r"(tmp0), "=r"(tmp1), "=r"(tmp2), "=r"(tmp3) : "l"(src));return uint4{tmp0, tmp1, tmp2, tmp3};+#endif}__always_inline __device__ void store_nc(uint1* __restrict__ dst, const uint1& value) {+#if FREETOKEN_USE_ROCM+ __builtin_nontemporal_store(value.x, &dst->x);+#elseuint32_t tmp = value.x;asm volatile("st.global.wt.b32 [%0],%1;" ::"l"(dst), "r"(tmp));+#endif}__always_inline __device__ void store_nc(uint2* __restrict__ dst, const uint2& value) {+#if FREETOKEN_USE_ROCM+ __builtin_nontemporal_store(+ __builtin_bit_cast(native_uint2, value), reinterpret_cast<native_uint2*>(dst));+#elseuint32_t tmp0 = value.x;uint32_t tmp1 = value.y;asm volatile("st.global.wt.v2.b32 [%0],{%1,%2};" ::"l"(dst), "r"(tmp0), "r"(tmp1));+#endif}__always_inline __device__ void store_nc(uint4* __restrict__ dst, const uint4& value) {+#if FREETOKEN_USE_ROCM+ __builtin_nontemporal_store(+ __builtin_bit_cast(native_uint4, value), reinterpret_cast<native_uint4*>(dst));+#elseuint32_t tmp0 = value.x;uint32_t tmp1 = value.y;uint32_t tmp2 = value.z;uint32_t tmp3 = value.w;asm volatile("st.global.wt.v4.b32 [%0],{%1,%2,%3,%4};" ::"l"(dst), "r"(tmp0), "r"(tmp1), "r"(tmp2), "r"(tmp3));+#endif}__always_inline __device__ void wait_flag_clear(const int32_t* __restrict__ flag_ptr) {@@ -75,7 +110,7 @@ __always_inline __device__ void wait_flag_clear(const int32_t* __restrict__ flagauto* flag = reinterpret_cast<int*>(const_cast<int32_t*>(flag_ptr));uint32_t sleep_ns = 128;while (atomicAdd(flag, 0) > 0) {-#if __CUDA_ARCH__ >= 700+#if !FREETOKEN_USE_ROCM && __CUDA_ARCH__ >= 700__nanosleep(sleep_ns);#endifsleep_ns = sleep_ns < 2048 ? (sleep_ns << 1) : 2048;@@ -147,7 +182,7 @@ inline bool host_ptr_identity() {}inline void* device_alias(void* ptr, DLDevice dev) {- if (dev.device_type == kDLCUDA || host_ptr_identity()) {+ if (dev.device_type == kDLCUDA || dev.device_type == kDLROCM || host_ptr_identity()) {return ptr;}void* mapped = nullptr;@@ -269,7 +304,7 @@ inline auto get_sync_flag_ptr(auto flag_dtype = host::SymbolicDType{};host::TensorMatcher({1}).with_dtype<int32_t>(flag_dtype)- .with_device<kDLCUDA>(device)+ .with_device<kDLCUDA, kDLROCM>(device).verify(sync_flag);return static_cast<int32_t*>(sync_flag.data_ptr());}@@ -344,17 +379,17 @@ struct FastIndexCopyKernel {TensorMatcher({-1, D}).with_dtype(data_dtype)- .with_device<kDLCUDA, kDLCUDAHost, kDLCPU>()+ .with_device<kDLCUDA, kDLROCM, kDLCUDAHost, kDLROCMHost, kDLCPU>().verify(src);TensorMatcher({-1, D}).with_dtype(data_dtype)- .with_device<kDLCUDA, kDLCUDAHost, kDLCPU>()+ .with_device<kDLCUDA, kDLROCM, kDLCUDAHost, kDLROCMHost, kDLCPU>().verify(dst);TensorMatcher({L}).with_dtype<int32_t, int64_t>(indices_dtype)- .with_device<kDLCUDA>(device)+ .with_device<kDLCUDA, kDLROCM>(device).verify(src_indices).verify(dst_indices);@@ -363,7 +398,7 @@ struct FastIndexCopyKernel {const auto num_indices_tensor = num_indices.value();TensorMatcher({1}).with_dtype<int64_t>(num_indices_dtype)- .with_device<kDLCUDA>(device)+ .with_device<kDLCUDA, kDLROCM>(device).verify(num_indices_tensor);num_indices_data_ptr = static_cast<const int64_t*>(num_indices_tensor.data_ptr());@@ -529,14 +564,14 @@ struct MultiIndexCopyKernel {auto indices_dtype = SymbolicDType{};auto num_indices_dtype = SymbolicDType{};- TensorMatcher({B}).with_dtype<int64_t>(ptr_dtype).with_device<kDLCUDA>(device)+ TensorMatcher({B}).with_dtype<int64_t>(ptr_dtype).with_device<kDLCUDA, kDLROCM>(device).verify(dst_ptrs).verify(src_ptrs).verify(feat_bytes);- TensorMatcher({L}).with_dtype<int32_t, int64_t>(indices_dtype).with_device<kDLCUDA>(device)+ TensorMatcher({L}).with_dtype<int32_t, int64_t>(indices_dtype).with_device<kDLCUDA, kDLROCM>(device).verify(dst_indices).verify(src_indices);const int64_t* valid_length = nullptr;if (num_indices.has_value()) {- TensorMatcher({1}).with_dtype<int64_t>(num_indices_dtype).with_device<kDLCUDA>(device)+ TensorMatcher({1}).with_dtype<int64_t>(num_indices_dtype).with_device<kDLCUDA, kDLROCM>(device).verify(num_indices.value());valid_length = static_cast<const int64_t*>(num_indices.value().data_ptr());}diff --git a/python/freetoken/kernel/csrc/pinned_tensor.cpp b/python/freetoken/kernel/csrc/pinned_tensor.cppindex c3947adf..9355f57a 100644--- a/python/freetoken/kernel/csrc/pinned_tensor.cpp+++ b/python/freetoken/kernel/csrc/pinned_tensor.cpp@@ -1,5 +1,5 @@#include <cstdint>-#include <cuda_runtime_api.h>+#include <freetoken/hip_compat.h>#include <torch/extension.h>namespace {diff --git a/python/freetoken/kernel/pynccl.py b/python/freetoken/kernel/pynccl.pyindex 23ea5735..71bde734 100644--- a/python/freetoken/kernel/pynccl.py+++ b/python/freetoken/kernel/pynccl.py@@ -27,6 +27,7 @@ else:@functools.cachedef _load_nccl_module() -> Module:+ # TODO(ROCm): NCCL -> RCCL migration for multi-GPU tensor parallelism on AMD.return load_aot("pynccl", cuda_files=["pynccl.cu"], extra_ldflags=["-lnccl"])diff --git a/python/freetoken/kernel/triton/activation.py b/python/freetoken/kernel/triton/activation.pyindex 0e4d28c8..9645960d 100644--- a/python/freetoken/kernel/triton/activation.py+++ b/python/freetoken/kernel/triton/activation.py@@ -20,8 +20,9 @@ import tritonimport triton.language as tlfrom triton.language.extra import libdevicefrom triton.language.extra.cuda import gdc_wait, gdc_launch_dependents+from triton.language import target_info-from freetoken.utils.arch import is_sm90_supported+from freetoken.utils.arch import is_rocm, is_sm90_supportedSILU = 0GELU = 1@@ -50,6 +51,8 @@ def _pdl_supported() -> bool:@triton.jitdef _fast_tanh(x):+ if target_info.is_hip():+ return libdevice.tanh(x)# PTX tanh.approx.f32 — single HW op, matches flashinfer math::tanh.return tl.inline_asm_elementwise("tanh.approx.f32 $0, $1;", "=f,f", [x],@@ -59,6 +62,8 @@ def _fast_tanh(x):@triton.jitdef _fast_ex2(x):+ if target_info.is_hip():+ return libdevice.exp2(x)# PTX ex2.approx.f32 — matches __expf fast path used by flashinfer silu.return tl.inline_asm_elementwise("ex2.approx.f32 $0, $1;", "=f,f", [x],@@ -141,8 +146,9 @@ def _act_and_mul(block_d = min(triton.next_power_of_2(d), 1024 if M >= 4096 else 512)num_stages = 2 if block_d == 1024 else 3_act_and_mul_kernel[grid](- o2, x2, d, alpha, limit, ACT=kind, ENABLE_PDL=pdl, launch_pdl=pdl,+ o2, x2, d, alpha, limit, ACT=kind, ENABLE_PDL=pdl,BLOCK_D=block_d, num_warps=4, num_stages=num_stages,+ **({} if is_rocm() else {"launch_pdl": pdl}),)return outdiff --git a/python/freetoken/kernel/triton/e4m3_compat.py b/python/freetoken/kernel/triton/e4m3_compat.pyindex 1d9f744c..6ea0c753 100644--- a/python/freetoken/kernel/triton/e4m3_compat.py+++ b/python/freetoken/kernel/triton/e4m3_compat.py@@ -59,6 +59,10 @@ def e4m3_native() -> bool:if _native is None:if FORCE_EMU:_native = False+ elif torch.version.hip is not None:+ # ROCm reports gfx1101 as capability (11, 0), which is not a CUDA+ # compute capability and must not select the native fp8e4nv path.+ _native = Falseelse:from freetoken.gpu_select import assigned_visible_gpudiff --git a/python/freetoken/kernel/triton/norm.py b/python/freetoken/kernel/triton/norm.pyindex 3f95c29f..9071e1df 100644--- a/python/freetoken/kernel/triton/norm.py+++ b/python/freetoken/kernel/triton/norm.py@@ -30,7 +30,7 @@ import tritonimport triton.language as tlfrom triton.language.extra.cuda import gdc_launch_dependents, gdc_wait-from freetoken.utils.arch import is_sm90_supported+from freetoken.utils.arch import is_rocm, is_sm90_supported_HEUR = {"BLOCK": lambda a: triton.next_power_of_2(a["H"])}@@ -144,7 +144,8 @@ def _rmsnorm(input, weight, eps, out, gemma: bool):pdl = contig and is_sm90_supported()_rmsnorm_kernel[(A, B)](out, input, weight, eps, H, sxa, sxb, soa, sob,- CONTIG=contig, ENABLE_PDL=pdl, launch_pdl=pdl, GEMMA=gemma,+ CONTIG=contig, ENABLE_PDL=pdl, GEMMA=gemma,+ **({} if is_rocm() else {"launch_pdl": pdl}),num_warps=_num_warps(A * B), num_stages=1,)return out@@ -172,7 +173,8 @@ def _fused_add_rmsnorm(input, residual, weight, eps, gemma: bool):pdl = contig and is_sm90_supported()_fused_add_rmsnorm_kernel[(A, B)](input, residual, weight, eps, H, sxa, sxb, sra, srb,- CONTIG=contig, ENABLE_PDL=pdl, launch_pdl=pdl, GEMMA=gemma,+ CONTIG=contig, ENABLE_PDL=pdl, GEMMA=gemma,+ **({} if is_rocm() else {"launch_pdl": pdl}),num_warps=_num_warps(A * B), num_stages=1,)diff --git a/python/freetoken/kernel/utils.py b/python/freetoken/kernel/utils.pyindex cdd765c7..9af61526 100644--- a/python/freetoken/kernel/utils.py+++ b/python/freetoken/kernel/utils.py@@ -5,6 +5,7 @@ import importlibimport osimport pathlibimport re+from functools import cachefrom typing import TYPE_CHECKING, Iterator, List, NamedTuple, Tuple, TypeAlias, Unionif TYPE_CHECKING:@@ -20,15 +21,23 @@ _TRUE_VALUES = {"1", "true", "yes", "on"}DEFAULT_INCLUDE = [str(KERNEL_PATH / "include")]DEFAULT_CFLAGS = ["-std=c++20", "-O3"]DEFAULT_CUDA_CFLAGS = ["-std=c++20", "-O3", "--expt-relaxed-constexpr"]+DEFAULT_HIP_CFLAGS = ["-std=c++20", "-O3"]DEFAULT_LDFLAGS = []-ARCH_LIST_ENV = "TVM_FFI_CUDA_ARCH_LIST"+CUDA_ARCH_LIST_ENV = "TVM_FFI_CUDA_ARCH_LIST"+ROCM_ARCH_LIST_ENV = "TVM_FFI_ROCM_ARCH_LIST"+++def _is_rocm() -> bool:+ import torch++ return getattr(torch.version, "hip", None) is not Nonedef _cuda_arch_list() -> List[str]:"""Archs a CUDA build targets: the AOT build's TVM_FFI_CUDA_ARCH_LIST, else the GPU this process is bound to."""- arch_list = os.getenv(ARCH_LIST_ENV, "").split()+ arch_list = os.getenv(CUDA_ARCH_LIST_ENV, "").split()if arch_list:return arch_listimport torch@@ -39,17 +48,34 @@ def _cuda_arch_list() -> List[str]:return [f"{major}.{minor}"]+def _rocm_arch_list() -> List[str]:+ """The single ROCm GPU this process is bound to, or an explicit cross-compile target."""+ from freetoken.utils.arch import get_rocm_gfx_arch++ arch = get_rocm_gfx_arch()+ if arch is None:+ raise RuntimeError(+ "Could not determine the bound ROCm GPU architecture; select a visible GPU "+ "or set FREETOKEN_ROCM_ARCH for cross-compilation"+ )+ return [arch]++@contextlib.contextmanager-def _pin_tvm_ffi_arch_ctx(arch_list: List[str]) -> Iterator[None]:- """Hand tvm-ffi the arch list through its env var for one build. Left unset, tvm-ffi asks nvidia-smi and takes the first GPU listed, which under --gpu or CUDA_VISIBLE_DEVICES on a mixed box is not the bound one."""- if os.getenv(ARCH_LIST_ENV) or not arch_list:+def _pin_tvm_ffi_arch_ctx(arch_list: List[str], env_var: str) -> Iterator[None]:+ """Pin tvm-ffi to the resolved targets for one build, then restore the caller's environment."""+ if not arch_list:yieldreturn- os.environ[ARCH_LIST_ENV] = " ".join(arch_list)+ old_arch_list = os.environ.get(env_var)+ os.environ[env_var] = " ".join(arch_list)try:yieldfinally:- os.environ.pop(ARCH_LIST_ENV, None)+ if old_arch_list is None:+ os.environ.pop(env_var, None)+ else:+ os.environ[env_var] = old_arch_listdef _cuda_cflags(extra: List[str], arch_list: List[str]) -> List[str]:@@ -63,6 +89,61 @@ def _cuda_cflags(extra: List[str], arch_list: List[str]) -> List[str]:cc = max(arch_list, key=_rank).rstrip("a").replace(".", "")flags = flags + [f"-gencode=arch=compute_{cc},code=compute_{cc}"]return flags+++def _hip_cflags(extra: List[str], arch_list: List[str]) -> List[str]:+ """HIP flags for a kernel build on ROCm."""+ # TODO(ROCm): Triton autotune configs need RDNA-specific tuning (wave count, LDS size).+ return DEFAULT_HIP_CFLAGS + extra + [f"--offload-arch={arch}" for arch in arch_list]+++@cache+def _rocm_link_flags() -> List[str]:+ """Make ROCm's runtime library discoverable to JIT link commands.++ Traditional ROCm installs provide ``libamdhip64.so`` under ``$ROCM_HOME/lib``.+ ROCm 7.14 Python SDK images only provide the versioned soname, while TVM-FFI+ still links with ``-lamdhip64``. Supply a cache-local unversioned symlink via+ an explicit linker search path without modifying the Python environment.+ """+ candidates: list[pathlib.Path] = []+ if os.getenv("ROCM_HOME"):+ candidates.append(pathlib.Path(os.environ["ROCM_HOME"]))+ try:+ from torch.utils.cpp_extension import ROCM_HOME++ if ROCM_HOME:+ candidates.append(pathlib.Path(ROCM_HOME))+ except ImportError:+ pass+ spec = importlib.util.find_spec("_rocm_sdk_core")+ if spec and spec.submodule_search_locations:+ candidates.append(pathlib.Path(next(iter(spec.submodule_search_locations))))+ candidates.append(pathlib.Path("/opt/rocm"))++ for rocm_home in dict.fromkeys(candidates):+ library_dir = rocm_home / "lib"+ unversioned = library_dir / "libamdhip64.so"+ link_dir = library_dir+ if not unversioned.exists():+ versioned = sorted(library_dir.glob("libamdhip64.so.*"))+ if not versioned:+ continue+ link_dir = pathlib.Path.home() / ".cache" / "freetoken" / "rocm-lib"+ link_dir.mkdir(parents=True, exist_ok=True)+ compat_link = link_dir / "libamdhip64.so"+ if not compat_link.exists() and not compat_link.is_symlink():+ try:+ compat_link.symlink_to(versioned[-1])+ except FileExistsError:+ # Multiple tensor-parallel ranks may prepare the same cache.+ pass++ return [f"-L{link_dir}", f"-Wl,-rpath,{library_dir}"]++ raise RuntimeError("Unable to locate libamdhip64 for ROCm JIT linking")++CPP_TEMPLATE_TYPE: TypeAlias = Union[int, float, bool]@@ -223,12 +304,16 @@ def load_aot(if prebuilt is not None:return prebuilt+ is_rocm = _is_rocm()arch_list: List[str] = []if cuda_files:- from freetoken.kernel._toolchain import check_nvcc_matches_torch+ if is_rocm:+ arch_list = _rocm_arch_list()+ else:+ from freetoken.kernel._toolchain import check_nvcc_matches_torch- check_nvcc_matches_torch()- arch_list = _cuda_arch_list()+ check_nvcc_matches_torch()+ arch_list = _cuda_arch_list()from tvm_ffi.cpp import load@@ -242,14 +327,23 @@ def load_aot(cpp_files = [str((KERNEL_PATH / "src" / f).resolve()) for f in cpp_files]cuda_files = [str((KERNEL_PATH / "src" / f).resolve()) for f in cuda_files]- with _pin_tvm_ffi_arch_ctx(arch_list):+ if is_rocm:+ cuda_cflags = _hip_cflags(extra_cuda_cflags, arch_list)+ runtime_ldflags = _rocm_link_flags()+ arch_list_env = ROCM_ARCH_LIST_ENV+ else:+ cuda_cflags = _cuda_cflags(extra_cuda_cflags, arch_list)+ runtime_ldflags = []+ arch_list_env = CUDA_ARCH_LIST_ENV++ with _pin_tvm_ffi_arch_ctx(arch_list, arch_list_env):return load(name,cpp_files=cpp_files,cuda_files=cuda_files,extra_cflags=DEFAULT_CFLAGS + extra_cflags,- extra_cuda_cflags=_cuda_cflags(extra_cuda_cflags, arch_list),- extra_ldflags=DEFAULT_LDFLAGS + extra_ldflags,+ extra_cuda_cflags=cuda_cflags,+ extra_ldflags=DEFAULT_LDFLAGS + runtime_ldflags + extra_ldflags,extra_include_paths=DEFAULT_INCLUDE + extra_include_paths,build_directory=build_directory,)@@ -272,12 +366,16 @@ def load_jit(if prebuilt is not None:return prebuilt+ is_rocm = _is_rocm()arch_list: List[str] = []if cuda_files or cuda_wrappers:- from freetoken.kernel._toolchain import check_nvcc_matches_torch+ if is_rocm:+ arch_list = _rocm_arch_list()+ else:+ from freetoken.kernel._toolchain import check_nvcc_matches_torch- check_nvcc_matches_torch()- arch_list = _cuda_arch_list()+ check_nvcc_matches_torch()+ arch_list = _cuda_arch_list()from tvm_ffi.cpp import load_inline@@ -300,14 +398,23 @@ def load_jit(cuda_sources = [f'#include "{path}"' for path in cuda_paths]cuda_sources += [_make_wrapper(tup) for tup in cuda_wrappers]- with _pin_tvm_ffi_arch_ctx(arch_list):+ if is_rocm:+ cuda_cflags = _hip_cflags(extra_cuda_cflags, arch_list)+ runtime_ldflags = _rocm_link_flags()+ arch_list_env = ROCM_ARCH_LIST_ENV+ else:+ cuda_cflags = _cuda_cflags(extra_cuda_cflags, arch_list)+ runtime_ldflags = []+ arch_list_env = CUDA_ARCH_LIST_ENV++ with _pin_tvm_ffi_arch_ctx(arch_list, arch_list_env):return load_inline(name,cpp_sources=cpp_sources,cuda_sources=cuda_sources,extra_cflags=DEFAULT_CFLAGS + extra_cflags,- extra_cuda_cflags=_cuda_cflags(extra_cuda_cflags, arch_list),- extra_ldflags=DEFAULT_LDFLAGS + extra_ldflags,+ extra_cuda_cflags=cuda_cflags,+ extra_ldflags=DEFAULT_LDFLAGS + runtime_ldflags + extra_ldflags,extra_include_paths=DEFAULT_INCLUDE + extra_include_paths,build_directory=build_directory,)diff --git a/python/freetoken/utils/__init__.py b/python/freetoken/utils/__init__.pyindex 2e4ad15f..af68bf58 100644--- a/python/freetoken/utils/__init__.py+++ b/python/freetoken/utils/__init__.py@@ -1,5 +1,7 @@from .arch import (is_arch_supported,+ is_rocm,+ get_rocm_gfx_arch,is_sm90_family,is_sm90_supported,is_sm100_family,@@ -35,6 +37,8 @@ __all__ = ["load_toolcall_anchor_id","init_logger","is_arch_supported",+ "is_rocm",+ "get_rocm_gfx_arch","is_sm90_family","is_sm90_supported","is_sm100_family",diff --git a/python/freetoken/utils/arch.py b/python/freetoken/utils/arch.pyindex 8c1c6c3d..30450596 100644--- a/python/freetoken/utils/arch.py+++ b/python/freetoken/utils/arch.py@@ -1,14 +1,66 @@from __future__ import annotationsimport functools+import os+import refrom typing import Tuple+_GFX_ARCH_RE = re.compile(r"gfx\d+[a-z]?")+++def _gfx_arch_from(value: object) -> str | None:+ match = _GFX_ARCH_RE.search(str(value).lower())+ return match.group(0) if match else None+++def is_rocm() -> bool:+ """True when torch is built for ROCm (AMD GPU) instead of CUDA."""+ import torch+ return getattr(torch.version, "hip", None) is not None+++def get_rocm_gfx_arch() -> str | None:+ """Return the current AMD GPU target (for example ``gfx1201``).++ Prefer the runtime device because build variables may contain multiple+ semicolon-separated targets. Environment variables remain useful for+ cross-compilation and systems where no GPU is currently visible; in that+ fallback mode the first target is returned. The result is process-cached+ for FreeToken's one-process-per-GPU execution model, so callers must select+ the intended device before the first call.+ """+ if not is_rocm():+ return None++ import torch++ if torch.cuda.is_available():+ try:+ props = torch.cuda.get_device_properties(torch.cuda.current_device())+ for attr in ("gcnArchName", "arch"):+ arch = _gfx_arch_from(getattr(props, attr, ""))+ if arch:+ return arch+ except (AttributeError, RuntimeError):+ pass++ for env_var in ("FREETOKEN_ROCM_ARCH", "PYTORCH_ROCM_ARCH", "HCC_AMDGPU_TARGET"):+ arch = _gfx_arch_from(os.getenv(env_var, ""))+ if arch:+ return arch+ return None++@functools.cachedef _get_torch_cuda_version() -> Tuple[int, int] | None:import torchimport torch.version+ if is_rocm():+ return Noneif not torch.cuda.is_available() or not torch.version.cuda:return Nonereturn torch.cuda.get_device_capability()diff --git a/setup.py b/setup.pyindex 7132eea0..997745f1 100644--- a/setup.py+++ b/setup.py@@ -1,15 +1,17 @@from __future__ import annotationsimport importlib.util+import osfrom pathlib import Pathimport sysfrom setuptools import setup-from torch.utils.cpp_extension import BuildExtension, CUDA_HOME, CppExtension+from torch.utils.cpp_extension import BuildExtension, CUDA_HOME, CppExtension, ROCM_HOMEROOT = Path(__file__).parent+KERNEL_INCLUDE = str(ROOT / "python" / "freetoken" / "kernel" / "csrc" / "include")def _check_toolchain() -> None:@@ -20,6 +22,43 @@ def _check_toolchain() -> None:module.check_nvcc_matches_torch()+def _is_rocm() -> bool:+ import torch+ return getattr(torch.version, "hip", None) is not None+++def _rocm_paths() -> tuple[list[str], list[str], str]:+ candidates: list[Path] = []+ if os.getenv("ROCM_HOME"):+ candidates.append(Path(os.environ["ROCM_HOME"]))+ if ROCM_HOME:+ candidates.append(Path(ROCM_HOME))++ # ROCm 7.14 PyTorch images ship the SDK as a Python package instead of+ # installing it at /opt/rocm.+ spec = importlib.util.find_spec("_rocm_sdk_core")+ if spec and spec.submodule_search_locations:+ candidates.append(Path(next(iter(spec.submodule_search_locations))))+ candidates.append(Path("/opt/rocm"))++ for rocm_home in dict.fromkeys(candidates):+ include_dir = rocm_home / "include"+ library_dir = rocm_home / "lib"+ if not (include_dir / "hip" / "hip_runtime.h").exists():+ continue+ if (library_dir / "libamdhip64.so").exists():+ return [str(include_dir)], [str(library_dir)], "amdhip64"+ versioned = sorted(library_dir.glob("libamdhip64.so.*"))+ if versioned:+ return [str(include_dir)], [str(library_dir)], f":{versioned[-1].name}"++ searched = ", ".join(str(path) for path in dict.fromkeys(candidates))+ raise RuntimeError(+ "A ROCm SDK with HIP headers and libamdhip64 is required to build on ROCm; "+ f"searched: {searched}. Set ROCM_HOME to override."+ )++def _cuda_runtime_paths() -> tuple[list[str], list[str]]:if CUDA_HOME is None:raise RuntimeError(@@ -33,7 +72,21 @@ def _cuda_runtime_paths() -> tuple[list[str], list[str]]:return [str(cuda_home / "include")], library_dirs-cuda_include_dirs, cuda_library_dirs = _cuda_runtime_paths()+IS_ROCM = _is_rocm()++if IS_ROCM:+ runtime_include_dirs, runtime_library_dirs, runtime_lib = _rocm_paths()+ runtime_link_args = [f"-Wl,-rpath,{runtime_library_dirs[0]}"]+ # These extensions contain host code only. BuildExtension supplies the ROCm+ # platform defines to the C++ compiler; offload architecture flags belong on+ # HIP device sources and would be rejected by the host compiler here.+ extra_compile = ["-O3", "-std=c++17"]+else:+ runtime_include_dirs, runtime_library_dirs = _cuda_runtime_paths()+ runtime_lib = "cudart"+ runtime_link_args = []+ extra_compile = ["-O3", "-std=c++17"]+_check_toolchain()@@ -44,12 +97,13 @@ setup(sources=["python/freetoken/kernel/csrc/pinned_tensor.cpp",],- include_dirs=cuda_include_dirs,- library_dirs=cuda_library_dirs,- libraries=["cudart"],- extra_compile_args=["-O3", "-std=c++17"],+ include_dirs=[KERNEL_INCLUDE, *runtime_include_dirs],+ library_dirs=runtime_library_dirs,+ libraries=[runtime_lib],+ extra_compile_args=extra_compile,+ extra_link_args=runtime_link_args,),- # CPU-compute MoE executor for --moe-backend cpu. Links cudart for the+ # CPU-compute MoE executor for --moe-backend cpu. Links cudart/amdhip64 for the# cudaLaunchHostFunc submit/sync graph nodes; the bf16 GEMV microkernels# use per-function target attributes (avx512bf16/avx512f) + a runtime# __builtin_cpu_supports dispatch, so the single binary stays portable@@ -59,10 +113,11 @@ setup(sources=["python/freetoken/kernel/csrc/cpu_moe/cpu_moe_ext.cpp",],- include_dirs=cuda_include_dirs,- library_dirs=cuda_library_dirs,- libraries=["cudart"],- extra_compile_args=["-O3", "-std=c++17", "-pthread"],+ include_dirs=[KERNEL_INCLUDE, *runtime_include_dirs],+ library_dirs=runtime_library_dirs,+ libraries=[runtime_lib],+ extra_compile_args=extra_compile + ["-pthread"],+ extra_link_args=runtime_link_args,),# disk-backed row store (PLE / Engram tables); Linux-only until the TableFile/BatchReader seams grow Windows bodies*([diff --git a/tests/kernels/test_e4m3_compat.py b/tests/kernels/test_e4m3_compat.pyindex 8bcb8e58..fa8a1f1d 100644--- a/tests/kernels/test_e4m3_compat.py+++ b/tests/kernels/test_e4m3_compat.py@@ -41,6 +41,29 @@ def _native_cc() -> bool:return torch.cuda.get_device_capability() >= (8, 9)[email protected](torch.version.hip is None, reason="needs ROCm")+def test_rocm_host_and_compile_time_native_gates_match():+ """The host wrapper and Triton constexpr must choose the same buffer ABI."""+ import triton+ import triton.language as tl++ from freetoken.kernel.triton.e4m3_compat import e4m3_native, e4m3_native_cx++ @triton.jit+ def gate_kernel(output):+ if e4m3_native_cx():+ value = 1+ else:+ value = 0+ tl.store(output, value)++ output = torch.empty(1, dtype=torch.int32, device="cuda")+ gate_kernel[(1,)](output)++ assert e4m3_native() is False+ assert output.item() == int(e4m3_native())++# ======================================================================================# 1. Primitives vs the native fp8 unit (needs sm_89+ hardware for the reference).# ======================================================================================diff --git a/tests/kernels/test_pinned_tensor.py b/tests/kernels/test_pinned_tensor.pyindex e61108fd..2fee4f25 100644--- a/tests/kernels/test_pinned_tensor.py+++ b/tests/kernels/test_pinned_tensor.py@@ -122,7 +122,12 @@ def test_host_device_ptr_is_identity_under_uva():pytest.skip("non-UVA platform: host_device_ptr rejects unregistered memory instead")# Under UVA cudaHostGetDevicePointer degenerates to identity for any host pointer# (no registration validation); rejection of pageable memory only exists on- # non-identity platforms (Windows/WDDM), where the translation is real.+ # non-identity CUDA platforms (Windows/WDDM), where the translation is real.+ # HIP validates registration even though registered/pinned memory uses the+ # identity address on Linux. Calling it with pageable memory also leaves a+ # sticky HIP error, so the pinned identity case above is the relevant check.+ if torch.version.hip is not None:+ returnpageable = torch.empty(64, dtype=torch.uint8)ext = _load_pinned_extension()assert ext.host_device_ptr(pageable.data_ptr()) == pageable.data_ptr()diff --git a/tests/kernels/test_rocm_launch_kwargs.py b/tests/kernels/test_rocm_launch_kwargs.pynew file mode 100644index 00000000..52fac545--- /dev/null+++ b/tests/kernels/test_rocm_launch_kwargs.py@@ -0,0 +1,51 @@+import pytest+import torch+++pytestmark = pytest.mark.skipif(+ not torch.cuda.is_available() or torch.version.hip is None,+ reason="a ROCm GPU is required",+)+++class _KernelRecorder:+ def __init__(self):+ self.kwargs = None++ def __getitem__(self, _grid):+ def launch(*_args, **kwargs):+ self.kwargs = kwargs++ return launch+++def test_rocm_activation_launch_omits_cuda_pdl_attribute(monkeypatch):+ import freetoken.kernel.triton.activation as activation++ recorder = _KernelRecorder()+ monkeypatch.setattr(activation, "_act_and_mul_kernel", recorder)+ monkeypatch.setattr(activation, "_pdl_supported", lambda: True)+ monkeypatch.setattr(activation, "is_rocm", lambda: True)++ activation.silu_and_mul(torch.ones((2, 128), device="cuda"))++ assert recorder.kwargs is not None+ assert recorder.kwargs["ENABLE_PDL"] is True+ assert "launch_pdl" not in recorder.kwargs+++def test_rocm_norm_launch_omits_cuda_pdl_attribute(monkeypatch):+ import freetoken.kernel.triton.norm as norm++ recorder = _KernelRecorder()+ monkeypatch.setattr(norm, "_rmsnorm_kernel", recorder)+ monkeypatch.setattr(norm, "is_sm90_supported", lambda: True)+ monkeypatch.setattr(norm, "is_rocm", lambda: True)++ x = torch.ones((2, 64), device="cuda")+ weight = torch.ones(64, device="cuda")+ norm.rmsnorm(x, weight)++ assert recorder.kwargs is not None+ assert recorder.kwargs["ENABLE_PDL"] is True+ assert "launch_pdl" not in recorder.kwargsdiff --git a/tests/utils/test_rocm_arch.py b/tests/utils/test_rocm_arch.pynew file mode 100644index 00000000..fcfab758--- /dev/null+++ b/tests/utils/test_rocm_arch.py@@ -0,0 +1,127 @@+import importlib+import os+import pathlib+import sys+import types+from types import SimpleNamespace++import torch++from freetoken.utils import arch+++def _clear_arch_caches() -> None:+ arch.get_rocm_gfx_arch.cache_clear()+++def test_rocm_arch_prefers_visible_device_over_multi_arch_build_env(monkeypatch):+ monkeypatch.setattr(arch, "is_rocm", lambda: True)+ monkeypatch.setattr(torch.cuda, "is_available", lambda: True)+ monkeypatch.setattr(torch.cuda, "current_device", lambda: 0)+ monkeypatch.setattr(+ torch.cuda,+ "get_device_properties",+ lambda _device: SimpleNamespace(gcnArchName="gfx1201:sramecc-:xnack-"),+ )+ monkeypatch.setenv("FREETOKEN_ROCM_ARCH", "gfx1100;gfx1200")+ _clear_arch_caches()++ assert arch.get_rocm_gfx_arch() == "gfx1201"++ _clear_arch_caches()+++def test_rocm_arch_falls_back_to_cross_compile_env(monkeypatch):+ monkeypatch.setattr(arch, "is_rocm", lambda: True)+ monkeypatch.setattr(torch.cuda, "is_available", lambda: False)+ monkeypatch.setenv("FREETOKEN_ROCM_ARCH", "gfx1200;gfx1201")+ _clear_arch_caches()++ assert arch.get_rocm_gfx_arch() == "gfx1200"++ _clear_arch_caches()+++def test_hip_cflags_target_only_resolved_arch(monkeypatch):+ from freetoken.kernel import utils++ monkeypatch.setenv("FREETOKEN_ROCM_ARCH", "gfx1200;gfx1201")+ monkeypatch.setattr(arch, "get_rocm_gfx_arch", lambda: "gfx1201")++ arch_list = utils._rocm_arch_list()+ flags = utils._hip_cflags(["-Wno-unused-command-line-argument"], arch_list)++ assert arch_list == ["gfx1201"]+ assert "--offload-arch=gfx1201" in flags+ assert "--offload-arch=gfx1200" not in flags+++def test_rocm_loaders_pin_tvm_ffi_to_resolved_arch(monkeypatch):+ from freetoken.kernel import utils++ builds = []++ def record_build(*_args, **kwargs):+ builds.append(+ (+ os.environ[utils.ROCM_ARCH_LIST_ENV],+ kwargs["extra_cuda_cflags"],+ )+ )+ return object()++ tvm_ffi = types.ModuleType("tvm_ffi")+ tvm_ffi.__path__ = []+ tvm_ffi_cpp = types.ModuleType("tvm_ffi.cpp")+ tvm_ffi_cpp.load = record_build+ tvm_ffi_cpp.load_inline = record_build+ monkeypatch.setitem(sys.modules, "tvm_ffi", tvm_ffi)+ monkeypatch.setitem(sys.modules, "tvm_ffi.cpp", tvm_ffi_cpp)+ monkeypatch.setenv(utils.DISABLE_KERNEL_CACHE_ENV, "1")+ monkeypatch.setenv(utils.ROCM_ARCH_LIST_ENV, "gfx1100 gfx1200")+ monkeypatch.setattr(utils, "_is_rocm", lambda: True)+ monkeypatch.setattr(utils, "_rocm_arch_list", lambda: ["gfx1201"])+ monkeypatch.setattr(utils, "_rocm_link_flags", lambda: [])++ utils.load_aot("test_rocm_aot_arch", cuda_files=["unused.cu"])+ utils.load_jit("test_rocm_jit_arch", cuda_files=["unused.cu"])++ assert builds == [+ ("gfx1201", [*utils.DEFAULT_HIP_CFLAGS, "--offload-arch=gfx1201"]),+ ("gfx1201", [*utils.DEFAULT_HIP_CFLAGS, "--offload-arch=gfx1201"]),+ ]+ assert os.environ[utils.ROCM_ARCH_LIST_ENV] == "gfx1100 gfx1200"+++def test_rocm_link_flags_support_versioned_modular_sdk(monkeypatch, tmp_path):+ import torch.utils.cpp_extension as cpp_extension++ from freetoken.kernel import utils++ sdk = tmp_path / "sdk"+ library_dir = sdk / "lib"+ library_dir.mkdir(parents=True)+ versioned_runtime = library_dir / "libamdhip64.so.7"+ versioned_runtime.write_bytes(b"")+ real_find_spec = importlib.util.find_spec++ def find_spec(name: str):+ if name == "_rocm_sdk_core":+ return SimpleNamespace(submodule_search_locations=[str(sdk)])+ return real_find_spec(name)++ monkeypatch.delenv("ROCM_HOME", raising=False)+ monkeypatch.setattr(cpp_extension, "ROCM_HOME", None)+ monkeypatch.setattr(importlib.util, "find_spec", find_spec)+ monkeypatch.setattr(pathlib.Path, "home", lambda: tmp_path)+ utils._rocm_link_flags.cache_clear()++ flags = utils._rocm_link_flags()++ compat_dir = tmp_path / ".cache" / "freetoken" / "rocm-lib"+ compat_link = compat_dir / "libamdhip64.so"+ assert f"-L{compat_dir}" in flags+ assert f"-Wl,-rpath,{library_dir}" in flags+ assert compat_link.resolve() == versioned_runtime.resolve()++ utils._rocm_link_flags.cache_clear()