Get up and running with Kimi, GLM, MiniMax, DeepSeek, gpt-oss, Qwen, Gemma and other models.
Add gfx1200 & gfx1201 support on linux (#9878)
saman-amdead27aa9fe85
diff --git a/CMakeLists.txt b/CMakeLists.txtindex 034fc7d79..e2447f32a 100644--- a/CMakeLists.txt+++ b/CMakeLists.txt@@ -86,9 +86,9 @@ if(CMAKE_CUDA_COMPILER))endif()-set(WINDOWS_AMDGPU_TARGETS_EXCLUDE_REGEX "^gfx(906|908|90a):xnack[+-]$"+set(WINDOWS_AMDGPU_TARGETS_EXCLUDE_REGEX "^gfx(906|908|90a|1200|1201):xnack[+-]$"CACHE STRING- "Regular expression describing AMDGPU_TARGETS not supported on Windows. Override to force building these targets. Default \"^gfx(906|908|90a):xnack[+-]$\"."+ "Regular expression describing AMDGPU_TARGETS not supported on Windows. Override to force building these targets. Default \"^gfx(906|908|90a|1200|1201):xnack[+-]$\".")check_language(HIP)@@ -97,7 +97,7 @@ if(CMAKE_HIP_COMPILER)find_package(hip REQUIRED)if(NOT AMDGPU_TARGETS)- list(FILTER AMDGPU_TARGETS INCLUDE REGEX "^gfx(900|94[012]|101[02]|1030|110[012])$")+ list(FILTER AMDGPU_TARGETS INCLUDE REGEX "^gfx(900|94[012]|101[02]|1030|110[012]|120[01])$")elseif(WIN32 AND WINDOWS_AMDGPU_TARGETS_EXCLUDE_REGEX)list(FILTER AMDGPU_TARGETS EXCLUDE REGEX ${WINDOWS_AMDGPU_TARGETS_EXCLUDE_REGEX})endif()diff --git a/CMakePresets.json b/CMakePresets.jsonindex f3507e3f2..bf99917f0 100644--- a/CMakePresets.json+++ b/CMakePresets.json@@ -56,7 +56,7 @@"name": "ROCm 6","inherits": [ "ROCm" ],"cacheVariables": {- "AMDGPU_TARGETS": "gfx900;gfx940;gfx941;gfx942;gfx1010;gfx1012;gfx1030;gfx1100;gfx1101;gfx1102;gfx1151;gfx906:xnack-;gfx908:xnack-;gfx90a:xnack+;gfx90a:xnack-"+ "AMDGPU_TARGETS": "gfx900;gfx940;gfx941;gfx942;gfx1010;gfx1012;gfx1030;gfx1100;gfx1101;gfx1102;gfx1151;gfx1200;gfx1201;gfx906:xnack-;gfx908:xnack-;gfx90a:xnack+;gfx90a:xnack-"}}],diff --git a/llama/patches/0022-add-rdna4-support.patch b/llama/patches/0022-add-rdna4-support.patchnew file mode 100644index 000000000..9b77ae84c--- /dev/null+++ b/llama/patches/0022-add-rdna4-support.patch@@ -0,0 +1,103 @@+From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001+From: Saman <[email protected]>+Date: Wed, 19 Mar 2025 14:02:26 -0700+Subject: [PATCH] add rdna4 support++---+ ggml/src/ggml-cuda/common.cuh | 6 ++++--+ ggml/src/ggml-cuda/mmq.cu | 2 +-+ ggml/src/ggml-cuda/mmq.cuh | 4 ++--+ ggml/src/ggml-cuda/mmvq.cu | 4 ++--+ ggml/src/ggml-cuda/vendors/hip.h | 4 +++++ 5 files changed, 13 insertions(+), 7 deletions(-)++diff --git a/ggml/src/ggml-cuda/common.cuh b/ggml/src/ggml-cuda/common.cuh+index adf0d3ec..b24593fc 100644+--- a/ggml/src/ggml-cuda/common.cuh++++ b/ggml/src/ggml-cuda/common.cuh+@@ -61,11 +61,13 @@+ #define GGML_CUDA_CC_RDNA1 (GGML_CUDA_CC_OFFSET_AMD + 0x1010) // RX 5000+ #define GGML_CUDA_CC_RDNA2 (GGML_CUDA_CC_OFFSET_AMD + 0x1030) // RX 6000, minimum for dp4a+ #define GGML_CUDA_CC_RDNA3 (GGML_CUDA_CC_OFFSET_AMD + 0x1100) // RX 7000, minimum for WMMA++#define GGML_CUDA_CC_RDNA4 (GGML_CUDA_CC_OFFSET_AMD + 0x1200) // RX 9000++ #define GGML_CUDA_CC_IS_RDNA(cc) (cc >= GGML_CUDA_CC_RDNA1)+ #define GGML_CUDA_CC_IS_RDNA1(cc) (cc >= GGML_CUDA_CC_RDNA1 && cc < GGML_CUDA_CC_RDNA2)+ #define GGML_CUDA_CC_IS_RDNA2(cc) (cc >= GGML_CUDA_CC_RDNA2 && cc < GGML_CUDA_CC_RDNA3)+-#define GGML_CUDA_CC_IS_RDNA3(cc) (cc >= GGML_CUDA_CC_RDNA3)++#define GGML_CUDA_CC_IS_RDNA3(cc) (cc >= GGML_CUDA_CC_RDNA3 && cc < GGML_CUDA_CC_RDNA4)++#define GGML_CUDA_CC_IS_RDNA4(cc) (cc >= GGML_CUDA_CC_RDNA4)+ #define GGML_CUDA_CC_IS_GCN(cc) (cc > GGML_CUDA_CC_OFFSET_AMD && cc < GGML_CUDA_CC_CDNA)+ #define GGML_CUDA_CC_IS_CDNA(cc) (cc >= GGML_CUDA_CC_CDNA && cc < GGML_CUDA_CC_RDNA1)++@@ -386,7 +388,7 @@ static __device__ __forceinline__ int ggml_cuda_dp4a(const int a, const int b, i+ #if defined(GGML_USE_HIP) && defined(__HIP_PLATFORM_AMD__)+ #if defined(__gfx906__) || defined(__gfx908__) || defined(__gfx90a__) || defined(RDNA2)+ c = __builtin_amdgcn_sdot4(a, b, c, false);+-#elif defined(RDNA3)++#elif defined(RDNA3) || defined(RDNA4)+ c = __builtin_amdgcn_sudot4( true, a, true, b, c, false);+ #elif defined(__gfx1010__) || defined(__gfx900__)+ int tmp1;+diff --git a/ggml/src/ggml-cuda/mmq.cu b/ggml/src/ggml-cuda/mmq.cu+index 10f2ebb1..933d945c 100644+--- a/ggml/src/ggml-cuda/mmq.cu++++ b/ggml/src/ggml-cuda/mmq.cu+@@ -149,5 +149,5 @@ bool ggml_cuda_should_use_mmq(enum ggml_type type, int cc, int64_t ne11) {+ return !fp16_mma_hardware_available(cc) || ne11 < MMQ_DP4A_MAX_BATCH_SIZE;+ }++- return (!GGML_CUDA_CC_IS_RDNA3(cc) && !GGML_CUDA_CC_IS_CDNA(cc)) || ne11 < MMQ_DP4A_MAX_BATCH_SIZE;++ return (!GGML_CUDA_CC_IS_RDNA4(cc) && !GGML_CUDA_CC_IS_RDNA3(cc) && !GGML_CUDA_CC_IS_CDNA(cc)) || ne11 < MMQ_DP4A_MAX_BATCH_SIZE;+ }+diff --git a/ggml/src/ggml-cuda/mmq.cuh b/ggml/src/ggml-cuda/mmq.cuh+index 0451c65f..66ce2bc9 100644+--- a/ggml/src/ggml-cuda/mmq.cuh++++ b/ggml/src/ggml-cuda/mmq.cuh+@@ -2577,9 +2577,9 @@ static __device__ void mul_mat_q_process_tile(++ template <ggml_type type, int mmq_x, int nwarps, bool need_check>+ #if defined(GGML_USE_HIP) && defined(__HIP_PLATFORM_AMD__)+-#if defined(RDNA3) || defined(RDNA2) || defined(CDNA) || defined(GCN)++#if defined(RDNA4) || defined(RDNA3) || defined(RDNA2) || defined(CDNA) || defined(GCN)+ __launch_bounds__(WARP_SIZE*nwarps, 2)+-#endif // defined(RDNA3) || defined(RDNA2) || defined(CDNA) || defined(GCN)++#endif // defined(RDNA4) || defined(RDNA3) || defined(RDNA2) || defined(CDNA) || defined(GCN)+ #else+ #if __CUDA_ARCH__ >= GGML_CUDA_CC_VOLTA+ __launch_bounds__(WARP_SIZE*nwarps, 1)+diff --git a/ggml/src/ggml-cuda/mmvq.cu b/ggml/src/ggml-cuda/mmvq.cu+index 4fb466ca..23ae7abc 100644+--- a/ggml/src/ggml-cuda/mmvq.cu++++ b/ggml/src/ggml-cuda/mmvq.cu+@@ -62,13 +62,13 @@ static __global__ void mul_mat_vec_q(++ constexpr vec_dot_q_cuda_t vec_dot_q_cuda = get_vec_dot_q_cuda(type);++-#if defined(GGML_USE_HIP) && defined(__HIP_PLATFORM_AMD__) && (defined(RDNA2) || defined(RDNA3))++#if defined(GGML_USE_HIP) && defined(__HIP_PLATFORM_AMD__) && (defined(RDNA2) || defined(RDNA3) || defined(RDNA4))+ constexpr int nwarps = 1;+ constexpr int rows_per_cuda_block = 1;+ #else+ constexpr int nwarps = ncols_y <= 4 ? 4 : 2;+ constexpr int rows_per_cuda_block = ncols_y == 1 ? 1 : 2;+-#endif // defined(GGML_USE_HIP) && defined(__HIP_PLATFORM_AMD__) && !defined(RDNA2) && !defined(RDNA3)++#endif // defined(GGML_USE_HIP) && defined(__HIP_PLATFORM_AMD__) && !defined(RDNA2) && !defined(RDNA3) && !defined(RDNA4)++ const int tid = WARP_SIZE*threadIdx.y + threadIdx.x;+ const int row0 = rows_per_cuda_block*blockIdx.x;+diff --git a/ggml/src/ggml-cuda/vendors/hip.h b/ggml/src/ggml-cuda/vendors/hip.h+index 81964611..a62544b5 100644+--- a/ggml/src/ggml-cuda/vendors/hip.h++++ b/ggml/src/ggml-cuda/vendors/hip.h+@@ -150,6 +150,10 @@+ #define CDNA+ #endif+++#if defined(__gfx1200__) || defined(__gfx1201__)++#define RDNA4++#endif+++ #if defined(__gfx1100__) || defined(__gfx1101__) || defined(__gfx1102__) || defined(__gfx1103__) || \+ defined(__gfx1150__) || defined(__gfx1151__)+ #define RDNA3diff --git a/ml/backend/ggml/ggml/src/ggml-cuda/common.cuh b/ml/backend/ggml/ggml/src/ggml-cuda/common.cuhindex adf0d3ecb..b24593fc2 100644--- a/ml/backend/ggml/ggml/src/ggml-cuda/common.cuh+++ b/ml/backend/ggml/ggml/src/ggml-cuda/common.cuh@@ -61,11 +61,13 @@#define GGML_CUDA_CC_RDNA1 (GGML_CUDA_CC_OFFSET_AMD + 0x1010) // RX 5000#define GGML_CUDA_CC_RDNA2 (GGML_CUDA_CC_OFFSET_AMD + 0x1030) // RX 6000, minimum for dp4a#define GGML_CUDA_CC_RDNA3 (GGML_CUDA_CC_OFFSET_AMD + 0x1100) // RX 7000, minimum for WMMA+#define GGML_CUDA_CC_RDNA4 (GGML_CUDA_CC_OFFSET_AMD + 0x1200) // RX 9000#define GGML_CUDA_CC_IS_RDNA(cc) (cc >= GGML_CUDA_CC_RDNA1)#define GGML_CUDA_CC_IS_RDNA1(cc) (cc >= GGML_CUDA_CC_RDNA1 && cc < GGML_CUDA_CC_RDNA2)#define GGML_CUDA_CC_IS_RDNA2(cc) (cc >= GGML_CUDA_CC_RDNA2 && cc < GGML_CUDA_CC_RDNA3)-#define GGML_CUDA_CC_IS_RDNA3(cc) (cc >= GGML_CUDA_CC_RDNA3)+#define GGML_CUDA_CC_IS_RDNA3(cc) (cc >= GGML_CUDA_CC_RDNA3 && cc < GGML_CUDA_CC_RDNA4)+#define GGML_CUDA_CC_IS_RDNA4(cc) (cc >= GGML_CUDA_CC_RDNA4)#define GGML_CUDA_CC_IS_GCN(cc) (cc > GGML_CUDA_CC_OFFSET_AMD && cc < GGML_CUDA_CC_CDNA)#define GGML_CUDA_CC_IS_CDNA(cc) (cc >= GGML_CUDA_CC_CDNA && cc < GGML_CUDA_CC_RDNA1)@@ -386,7 +388,7 @@ static __device__ __forceinline__ int ggml_cuda_dp4a(const int a, const int b, i#if defined(GGML_USE_HIP) && defined(__HIP_PLATFORM_AMD__)#if defined(__gfx906__) || defined(__gfx908__) || defined(__gfx90a__) || defined(RDNA2)c = __builtin_amdgcn_sdot4(a, b, c, false);-#elif defined(RDNA3)+#elif defined(RDNA3) || defined(RDNA4)c = __builtin_amdgcn_sudot4( true, a, true, b, c, false);#elif defined(__gfx1010__) || defined(__gfx900__)int tmp1;diff --git a/ml/backend/ggml/ggml/src/ggml-cuda/mmq.cu b/ml/backend/ggml/ggml/src/ggml-cuda/mmq.cuindex 10f2ebb1c..933d945c8 100644--- a/ml/backend/ggml/ggml/src/ggml-cuda/mmq.cu+++ b/ml/backend/ggml/ggml/src/ggml-cuda/mmq.cu@@ -149,5 +149,5 @@ bool ggml_cuda_should_use_mmq(enum ggml_type type, int cc, int64_t ne11) {return !fp16_mma_hardware_available(cc) || ne11 < MMQ_DP4A_MAX_BATCH_SIZE;}- return (!GGML_CUDA_CC_IS_RDNA3(cc) && !GGML_CUDA_CC_IS_CDNA(cc)) || ne11 < MMQ_DP4A_MAX_BATCH_SIZE;+ return (!GGML_CUDA_CC_IS_RDNA4(cc) && !GGML_CUDA_CC_IS_RDNA3(cc) && !GGML_CUDA_CC_IS_CDNA(cc)) || ne11 < MMQ_DP4A_MAX_BATCH_SIZE;}diff --git a/ml/backend/ggml/ggml/src/ggml-cuda/mmq.cuh b/ml/backend/ggml/ggml/src/ggml-cuda/mmq.cuhindex 0451c65f3..66ce2bc95 100644--- a/ml/backend/ggml/ggml/src/ggml-cuda/mmq.cuh+++ b/ml/backend/ggml/ggml/src/ggml-cuda/mmq.cuh@@ -2577,9 +2577,9 @@ static __device__ void mul_mat_q_process_tile(template <ggml_type type, int mmq_x, int nwarps, bool need_check>#if defined(GGML_USE_HIP) && defined(__HIP_PLATFORM_AMD__)-#if defined(RDNA3) || defined(RDNA2) || defined(CDNA) || defined(GCN)+#if defined(RDNA4) || defined(RDNA3) || defined(RDNA2) || defined(CDNA) || defined(GCN)__launch_bounds__(WARP_SIZE*nwarps, 2)-#endif // defined(RDNA3) || defined(RDNA2) || defined(CDNA) || defined(GCN)+#endif // defined(RDNA4) || defined(RDNA3) || defined(RDNA2) || defined(CDNA) || defined(GCN)#else#if __CUDA_ARCH__ >= GGML_CUDA_CC_VOLTA__launch_bounds__(WARP_SIZE*nwarps, 1)diff --git a/ml/backend/ggml/ggml/src/ggml-cuda/mmvq.cu b/ml/backend/ggml/ggml/src/ggml-cuda/mmvq.cuindex 4fb466ca0..23ae7abcb 100644--- a/ml/backend/ggml/ggml/src/ggml-cuda/mmvq.cu+++ b/ml/backend/ggml/ggml/src/ggml-cuda/mmvq.cu@@ -62,13 +62,13 @@ static __global__ void mul_mat_vec_q(constexpr vec_dot_q_cuda_t vec_dot_q_cuda = get_vec_dot_q_cuda(type);-#if defined(GGML_USE_HIP) && defined(__HIP_PLATFORM_AMD__) && (defined(RDNA2) || defined(RDNA3))+#if defined(GGML_USE_HIP) && defined(__HIP_PLATFORM_AMD__) && (defined(RDNA2) || defined(RDNA3) || defined(RDNA4))constexpr int nwarps = 1;constexpr int rows_per_cuda_block = 1;#elseconstexpr int nwarps = ncols_y <= 4 ? 4 : 2;constexpr int rows_per_cuda_block = ncols_y == 1 ? 1 : 2;-#endif // defined(GGML_USE_HIP) && defined(__HIP_PLATFORM_AMD__) && !defined(RDNA2) && !defined(RDNA3)+#endif // defined(GGML_USE_HIP) && defined(__HIP_PLATFORM_AMD__) && !defined(RDNA2) && !defined(RDNA3) && !defined(RDNA4)const int tid = WARP_SIZE*threadIdx.y + threadIdx.x;const int row0 = rows_per_cuda_block*blockIdx.x;diff --git a/ml/backend/ggml/ggml/src/ggml-cuda/vendors/hip.h b/ml/backend/ggml/ggml/src/ggml-cuda/vendors/hip.hindex 81964611c..a62544b5a 100644--- a/ml/backend/ggml/ggml/src/ggml-cuda/vendors/hip.h+++ b/ml/backend/ggml/ggml/src/ggml-cuda/vendors/hip.h@@ -150,6 +150,10 @@#define CDNA#endif+#if defined(__gfx1200__) || defined(__gfx1201__)+#define RDNA4+#endif+#if defined(__gfx1100__) || defined(__gfx1101__) || defined(__gfx1102__) || defined(__gfx1103__) || \defined(__gfx1150__) || defined(__gfx1151__)#define RDNA3