LLM inference in C/C++
C++ ggml
kleidiai: Add SME vs SME2 distinction in kernel dispatch (#25478)
Rajendra Matchaf15bd6090166
diff --git a/ggml/include/ggml-cpu.h b/ggml/include/ggml-cpu.hindex e3e067c916..dc6453c6ea 100644--- a/ggml/include/ggml-cpu.h+++ b/ggml/include/ggml-cpu.h@@ -100,6 +100,7 @@ extern "C" {GGML_BACKEND_API int ggml_cpu_has_sve (void);GGML_BACKEND_API int ggml_cpu_get_sve_cnt (void); // sve vector length in bytesGGML_BACKEND_API int ggml_cpu_has_sme (void);+ GGML_BACKEND_API int ggml_cpu_has_sme2 (void);// otherGGML_BACKEND_API int ggml_cpu_has_riscv_v (void);GGML_BACKEND_API int ggml_cpu_get_rvv_vlen (void); // risc-v vector length in bytesdiff --git a/ggml/src/ggml-cpu/CMakeLists.txt b/ggml/src/ggml-cpu/CMakeLists.txtindex 1316978e2e..836bae4d05 100644--- a/ggml/src/ggml-cpu/CMakeLists.txt+++ b/ggml/src/ggml-cpu/CMakeLists.txt@@ -678,7 +678,18 @@ function(ggml_add_cpu_backend_variant_impl tag_name)endif()if (NOT SME_ENABLED MATCHES -1)- list(APPEND GGML_KLEIDIAI_SOURCES+ list(APPEND GGML_KLEIDIAI_SME_SOURCES+ ${KLEIDIAI_SRC}/kai/ukernels/matmul/matmul_clamp_f32_qai8dxp_qsi8cxp/kai_matmul_clamp_f32_qai8dxp1vlx4_qsi8cxp4vlx4_1vlx4vl_sme_mopa.c+ ${KLEIDIAI_SRC}/kai/ukernels/matmul/matmul_clamp_f32_qai8dxp_qsi8cxp/kai_matmul_clamp_f32_qai8dxp1vlx4_qsi8cxp4vlx4_1vlx4vl_sme_mopa_asm.S+ ${KLEIDIAI_SRC}/kai/ukernels/matmul/matmul_clamp_f32_qai8dxp_qsi8cxp/kai_matmul_clamp_f32_qai8dxp1x4_qsi8cxp4vlx4_1x4vl_sme_dot.c+ ${KLEIDIAI_SRC}/kai/ukernels/matmul/matmul_clamp_f32_qai8dxp_qsi8cxp/kai_matmul_clamp_f32_qai8dxp1x4_qsi8cxp4vlx4_1x4vl_sme_dot_asm.S+ ${KLEIDIAI_SRC}/kai/ukernels/matmul/matmul_clamp_f32_f32p_f32p/kai_matmul_clamp_f32_f32p2vlx1_f32p2vlx1b_2vlx2vl_sme_mopa.c+ ${KLEIDIAI_SRC}/kai/ukernels/matmul/matmul_clamp_f32_f32p_f32p/kai_matmul_clamp_f32_f32p2vlx1_f32p2vlx1b_2vlx2vl_sme_mopa_asm.S)+ set_source_files_properties(${GGML_KLEIDIAI_SME_SOURCES}+ PROPERTIES COMPILE_OPTIONS "-fno-tree-vectorize;${ARCH_FLAGS_TEMP}+sve+sve2+sme")+ list(APPEND GGML_CPU_SOURCES ${GGML_KLEIDIAI_SME_SOURCES})++ list(APPEND GGML_KLEIDIAI_SME2_SOURCES${KLEIDIAI_SRC}/kai/ukernels/matmul/matmul_clamp_f32_qsi8d32p_qsi4c32p/kai_matmul_clamp_f32_qsi8d32p1x4_qsi4c32p4vlx4_1x4vl_sme2_sdot.c${KLEIDIAI_SRC}/kai/ukernels/matmul/matmul_clamp_f32_qai8dxp_qsi8cxp/kai_matmul_clamp_f32_qai8dxp1vlx4_qsi8cxp4vlx4_1vlx4vl_sme2_mopa.c${KLEIDIAI_SRC}/kai/ukernels/matmul/matmul_clamp_f32_qai8dxp_qsi8cxp/kai_matmul_clamp_f32_qai8dxp1vlx4_qsi8cxp4vlx4_1vlx4vl_sme2_mopa_asm.S@@ -698,7 +709,10 @@ function(ggml_add_cpu_backend_variant_impl tag_name)${KLEIDIAI_SRC}/kai/ukernels/matmul/pack/kai_rhs_pack_nxk_f32p2vlx1biasf32_f32_f32_sme.c${KLEIDIAI_SRC}/kai/ukernels/matmul/pack/kai_rhs_pack_nxk_f32p2vlx1biasf32_f32_f32_sme_asm.S${KLEIDIAI_SRC}/kai/kai_common_sme_asm.S)- set(PRIVATE_ARCH_FLAGS "-fno-tree-vectorize;${PRIVATE_ARCH_FLAGS}+sve+sve2+sme2+fp16")+ set_source_files_properties(${GGML_KLEIDIAI_SME2_SOURCES}+ PROPERTIES COMPILE_OPTIONS "-fno-tree-vectorize;${ARCH_FLAGS_TEMP}+sve+sve2+sme2+fp16")+ list(APPEND GGML_CPU_SOURCES ${GGML_KLEIDIAI_SME2_SOURCES})+ set(PRIVATE_ARCH_FLAGS "-fno-tree-vectorize;${PRIVATE_ARCH_FLAGS}")endif()if (NOT SVE_ENABLED MATCHES -1)diff --git a/ggml/src/ggml-cpu/arch/arm/cpu-feats.cpp b/ggml/src/ggml-cpu/arch/arm/cpu-feats.cppindex c460c54911..adfbd2e4e9 100644--- a/ggml/src/ggml-cpu/arch/arm/cpu-feats.cpp+++ b/ggml/src/ggml-cpu/arch/arm/cpu-feats.cpp@@ -28,6 +28,7 @@ struct aarch64_features {bool has_sve2 = false;bool has_i8mm = false;bool has_sme = false;+ bool has_sme2 = false;aarch64_features() {#if defined(__linux__)@@ -56,6 +57,10 @@ struct aarch64_features {has_sme = static_cast<bool>(oldp);}+ if (sysctlbyname("hw.optional.arm.FEAT_SME2", &oldp, &size, NULL, 0) == 0) {+ has_sme2 = static_cast<bool>(oldp);+ }+// Apple apparently does not implement SVE yet#endif}diff --git a/ggml/src/ggml-cpu/ggml-cpu.c b/ggml/src/ggml-cpu/ggml-cpu.cindex 0b8d9fac65..2664420e39 100644--- a/ggml/src/ggml-cpu/ggml-cpu.c+++ b/ggml/src/ggml-cpu/ggml-cpu.c@@ -3792,6 +3792,14 @@ int ggml_cpu_has_sme(void) {#endif}+int ggml_cpu_has_sme2(void) {+#if defined(__ARM_ARCH) && defined(__ARM_FEATURE_SME2)+ return 1;+#else+ return 0;+#endif+}+void ggml_cpu_init(void) {// needed to initialize ggml_time{diff --git a/ggml/src/ggml-cpu/ggml-cpu.cpp b/ggml/src/ggml-cpu/ggml-cpu.cppindex 49023ebf61..74631c2857 100644--- a/ggml/src/ggml-cpu/ggml-cpu.cpp+++ b/ggml/src/ggml-cpu/ggml-cpu.cpp@@ -595,6 +595,9 @@ static ggml_backend_feature * ggml_backend_cpu_get_features(ggml_backend_reg_t rif (ggml_cpu_has_sme()) {features.push_back({ "SME", "1" });}+ if (ggml_cpu_has_sme2()) {+ features.push_back({ "SME2", "1" });+ }if (ggml_cpu_has_riscv_v()) {features.push_back({ "RISCV_V", "1" });}diff --git a/ggml/src/ggml-cpu/kleidiai/kernels.cpp b/ggml/src/ggml-cpu/kleidiai/kernels.cppindex bf03fd766b..3c31ab9d35 100644--- a/ggml/src/ggml-cpu/kleidiai/kernels.cpp+++ b/ggml/src/ggml-cpu/kleidiai/kernels.cpp@@ -13,6 +13,8 @@#include "kai_matmul_clamp_f32_bf16p2vlx2_bf16p2vlx2_2vlx2vl_sme2_mopa.h"#include "kai_matmul_clamp_f32_qai8dxp1vlx4_qsi8cxp4vlx4_1vlx4vl_sme2_mopa.h"#include "kai_matmul_clamp_f32_qai8dxp1x4_qsi8cxp4vlx4_1x4vl_sme2_dot.h"+#include "kai_matmul_clamp_f32_qai8dxp1vlx4_qsi8cxp4vlx4_1vlx4vl_sme_mopa.h"+#include "kai_matmul_clamp_f32_qai8dxp1x4_qsi8cxp4vlx4_1x4vl_sme_dot.h"#include "kai_matmul_clamp_f32_qai8dxp1x8_qsi8cxp4x8_1x4_neon_dotprod.h"#include "kai_matmul_clamp_f32_qai8dxp1x4_qsi8cxp4x4_1x4_neon_dotprod.h"#include "kai_matmul_clamp_f32_qai8dxp4x4_qsi8cxp4x4_16x4_neon_dotprod.h"@@ -21,6 +23,7 @@#include "kai_matmul_clamp_f32_qsi8d32p1x8_qsi4c32p8x8_1x8_sve_dotprod.h"#include "kai_matmul_clamp_f32_f16p1vlx2_qsi4c32p4vlx2_1vlx4vl_sme2_mopa.h"#include "kai_matmul_clamp_f32_f32p2vlx1_f32p2vlx1biasf32_sme2_mopa.h"+#include "kai_matmul_clamp_f32_f32p2vlx1_f32p2vlx1b_2vlx2vl_sme_mopa.h"#include "kai_lhs_pack_bf16p2vlx2_f32_sme.h"#include "kai_lhs_pack_f32p2vlx1_f32_sme.h"@@ -359,7 +362,7 @@ static ggml_kleidiai_kernels gemm_gemv_kernels[] = {/* .packed_stride_ex = */ &rhs_stride_fn4<kai_get_rhs_packed_stride_rhs_pack_nxk_qsi4c32ps1s0scalef16_qsu4c32s16s0_neon>,/* .pack_func_ex = */ &rhs_pack_fn12<kai_run_rhs_pack_nxk_qsi4c32ps1s0scalef16_qsu4c32s16s0_neon>,},- /* .required_cpu = */ CPU_FEATURE_SME,+ /* .required_cpu = */ CPU_FEATURE_SME2,/* .lhs_type = */ GGML_TYPE_F32,/* .rhs_type = */ GGML_TYPE_Q4_0,/* .op_type = */ GGML_TYPE_F32,@@ -412,7 +415,7 @@ static ggml_kleidiai_kernels gemm_gemv_kernels[] = {/* .packed_stride_ex = */ &rhs_stride_fn1<kai_get_rhs_packed_stride_rhs_pack_kxn_bf16p2vlx2b_f32_x32_sme>,/* .pack_func_ex = */ &rhs_pack_fn13<kai_run_rhs_pack_kxn_bf16p2vlx2b_f32_x32_sme>,},- /* .required_cpu = */ CPU_FEATURE_SME,+ /* .required_cpu = */ CPU_FEATURE_SME2,/* .lhs_type = */ GGML_TYPE_F32,/* .rhs_type = */ GGML_TYPE_F16,/* .op_type = */ GGML_TYPE_F32,@@ -749,6 +752,59 @@ static ggml_kleidiai_kernels gemm_gemv_kernels_q8[] = {/* .packed_stride_ex = */ &rhs_stride_fn4<kai_get_rhs_packed_stride_rhs_pack_nxk_qsi8cxp_qsi8cx_neon>,/* .pack_func_ex = */ &rhs_pack_scale_fn12<kai_run_rhs_pack_nxk_qsi8cxp_qsi8cx_neon>,},+ /* .required_cpu = */ CPU_FEATURE_SME2,+ /* .lhs_type = */ GGML_TYPE_F32,+ /* .rhs_type = */ GGML_TYPE_Q8_0,+ /* .op_type = */ GGML_TYPE_F32,+ },+ {+ /* SME GEMM (pure SME, no SME2 required) */+ {+ /* .get_m_step = */ kai_get_m_step_matmul_clamp_f32_qai8dxp1vlx4_qsi8cxp4vlx4_1vlx4vl_sme_mopa,+ /* .get_n_step = */ kai_get_n_step_matmul_clamp_f32_qai8dxp1vlx4_qsi8cxp4vlx4_1vlx4vl_sme_mopa,+ /* .get_mr = */ kai_get_mr_matmul_clamp_f32_qai8dxp1vlx4_qsi8cxp4vlx4_1vlx4vl_sme_mopa,+ /* .get_nr = */ kai_get_nr_matmul_clamp_f32_qai8dxp1vlx4_qsi8cxp4vlx4_1vlx4vl_sme_mopa,+ /* .get_kr = */ kai_get_kr_matmul_clamp_f32_qai8dxp1vlx4_qsi8cxp4vlx4_1vlx4vl_sme_mopa,+ /* .get_sr = */ kai_get_sr_matmul_clamp_f32_qai8dxp1vlx4_qsi8cxp4vlx4_1vlx4vl_sme_mopa,+ /* .get_dst_offset = */ kai_get_dst_offset_matmul_clamp_f32_qai8dxp1vlx4_qsi8cxp4vlx4_1vlx4vl_sme_mopa,+ /* .get_dst_size = */ kai_get_dst_size_matmul_clamp_f32_qai8dxp1vlx4_qsi8cxp4vlx4_1vlx4vl_sme_mopa,+ /* .get_lhs_offset_ex = */ &kernel_offs_fn2<kai_get_lhs_packed_offset_matmul_clamp_f32_qai8dxp1vlx4_qsi8cxp4vlx4_1vlx4vl_sme_mopa>,+ /* .get_rhs_packed_offset_ex = */ &kernel_offs_fn2<kai_get_rhs_packed_offset_matmul_clamp_f32_qai8dxp1vlx4_qsi8cxp4vlx4_1vlx4vl_sme_mopa>,+ /* .run_kernel_ex = */ &kernel_run_float_fn10<kai_run_matmul_clamp_f32_qai8dxp1vlx4_qsi8cxp4vlx4_1vlx4vl_sme_mopa>,+ },+ /* .gemm_lhs_info = */ {+ /* .get_offset = */ kai_get_lhs_offset_lhs_quant_pack_qai8dxp_f32,+ /* .get_packed_offset_ex = */ &lhs_offs_fn5<kai_get_lhs_packed_offset_lhs_quant_pack_qai8dxp_f32>,+ /* .packed_size_ex = */ &lhs_ps_fn5<kai_get_lhs_packed_size_lhs_quant_pack_qai8dxp_f32>,+ /* .pack_func_ex = */ &lhs_pack_float_fn9_no_bl<kai_run_lhs_quant_pack_qai8dxp_f32>,+ },+ /* SME GEMV (pure SME, no SME2 required) */+ {+ /* .get_m_step = */ kai_get_m_step_matmul_clamp_f32_qai8dxp1x4_qsi8cxp4vlx4_1x4vl_sme_dot,+ /* .get_n_step = */ kai_get_n_step_matmul_clamp_f32_qai8dxp1x4_qsi8cxp4vlx4_1x4vl_sme_dot,+ /* .get_mr = */ kai_get_mr_matmul_clamp_f32_qai8dxp1x4_qsi8cxp4vlx4_1x4vl_sme_dot,+ /* .get_nr = */ kai_get_nr_matmul_clamp_f32_qai8dxp1x4_qsi8cxp4vlx4_1x4vl_sme_dot,+ /* .get_kr = */ kai_get_kr_matmul_clamp_f32_qai8dxp1x4_qsi8cxp4vlx4_1x4vl_sme_dot,+ /* .get_sr = */ kai_get_sr_matmul_clamp_f32_qai8dxp1x4_qsi8cxp4vlx4_1x4vl_sme_dot,+ /* .get_dst_offset = */ kai_get_dst_offset_matmul_clamp_f32_qai8dxp1x4_qsi8cxp4vlx4_1x4vl_sme_dot,+ /* .get_dst_size = */ kai_get_dst_size_matmul_clamp_f32_qai8dxp1x4_qsi8cxp4vlx4_1x4vl_sme_dot,+ /* .get_lhs_offset_ex = */ &kernel_offs_fn2<kai_get_lhs_packed_offset_matmul_clamp_f32_qai8dxp1x4_qsi8cxp4vlx4_1x4vl_sme_dot>,+ /* .get_rhs_packed_offset_ex = */ &kernel_offs_fn2<kai_get_rhs_packed_offset_matmul_clamp_f32_qai8dxp1x4_qsi8cxp4vlx4_1x4vl_sme_dot>,+ /* .run_kernel_ex = */ &kernel_run_float_fn10<kai_run_matmul_clamp_f32_qai8dxp1x4_qsi8cxp4vlx4_1x4vl_sme_dot>,+ },+ /* .gemv_lhs_info = */ {+ /* .get_offset = */ kai_get_lhs_offset_lhs_quant_pack_qai8dxp_f32,+ /* .get_packed_offset_ex = */ &lhs_offs_fn5<kai_get_lhs_packed_offset_lhs_quant_pack_qai8dxp_f32>,+ /* .packed_size_ex = */ &lhs_ps_fn5<kai_get_lhs_packed_size_lhs_quant_pack_qai8dxp_f32>,+ /* .pack_func_ex = */ &lhs_pack_float_fn9_no_bl<kai_run_lhs_quant_pack_qai8dxp_f32>,+ },+ /* .rhs_info = */ {+ /* .packed_stride = */ kai_get_rhs_packed_stride_rhs_pack_nxk_qsi8cxp_qsi8cx_neon,+ /* .to_float = */ dequantize_row_qsi8cxp,+ /* .packed_size_ex = */ &rhs_ps_fn5<kai_get_rhs_packed_size_rhs_pack_nxk_qsi8cxp_qsi8cx_neon>,+ /* .packed_stride_ex = */ &rhs_stride_fn4<kai_get_rhs_packed_stride_rhs_pack_nxk_qsi8cxp_qsi8cx_neon>,+ /* .pack_func_ex = */ &rhs_pack_scale_fn12<kai_run_rhs_pack_nxk_qsi8cxp_qsi8cx_neon>,+ },/* .required_cpu = */ CPU_FEATURE_SME,/* .lhs_type = */ GGML_TYPE_F32,/* .rhs_type = */ GGML_TYPE_Q8_0,@@ -871,7 +927,7 @@ static ggml_kleidiai_kernels gemm_gemv_kernels_q8[] = {static ggml_kleidiai_kernels ggml_kleidiai_kernels_f32[] = {#if defined(__ARM_FEATURE_SME){- /* SME GEMM */+ /* SME2 GEMM */{/* .get_m_step = */ kai_get_m_step_matmul_clamp_f32_f32p2vlx1_f32p2vlx1biasf32_sme2_mopa,/* .get_n_step = */ kai_get_n_step_matmul_clamp_f32_f32p2vlx1_f32p2vlx1biasf32_sme2_mopa,@@ -918,6 +974,59 @@ static ggml_kleidiai_kernels ggml_kleidiai_kernels_f32[] = {/* .packed_stride_ex = */ &rhs_stride_fn1<kai_get_rhs_packed_stride_rhs_pack_nxk_f32p2vlx1biasf32_f32_f32_sme>,/* .pack_func_ex = */ &rhs_pack_fn13<kai_run_rhs_pack_nxk_f32p2vlx1biasf32_f32_f32_sme>,},+ /* .required_cpu = */ CPU_FEATURE_SME2,+ /* .lhs_type = */ GGML_TYPE_F32,+ /* .rhs_type = */ GGML_TYPE_F32,+ /* .op_type = */ GGML_TYPE_F32,+ },+ {+ /* SME GEMM */+ {+ /* .get_m_step = */ kai_get_m_step_matmul_clamp_f32_f32p2vlx1_f32p2vlx1b_2vlx2vl_sme_mopa,+ /* .get_n_step = */ kai_get_n_step_matmul_clamp_f32_f32p2vlx1_f32p2vlx1b_2vlx2vl_sme_mopa,+ /* .get_mr = */ kai_get_mr_matmul_clamp_f32_f32p2vlx1_f32p2vlx1b_2vlx2vl_sme_mopa,+ /* .get_nr = */ kai_get_nr_matmul_clamp_f32_f32p2vlx1_f32p2vlx1b_2vlx2vl_sme_mopa,+ /* .get_kr = */ kai_get_kr_matmul_clamp_f32_f32p2vlx1_f32p2vlx1b_2vlx2vl_sme_mopa,+ /* .get_sr = */ kai_get_sr_matmul_clamp_f32_f32p2vlx1_f32p2vlx1b_2vlx2vl_sme_mopa,+ /* .get_dst_offset = */ kai_get_dst_offset_matmul_clamp_f32_f32p2vlx1_f32p2vlx1b_2vlx2vl_sme_mopa,+ /* .get_dst_size = */ kai_get_dst_size_matmul_clamp_f32_f32p2vlx1_f32p2vlx1b_2vlx2vl_sme_mopa,+ /* .get_lhs_offset_ex = */ &kernel_offs_fn2<kai_get_lhs_packed_offset_matmul_clamp_f32_f32p2vlx1_f32p2vlx1b_2vlx2vl_sme_mopa>,+ /* .get_rhs_packed_offset_ex = */ &kernel_offs_fn2<kai_get_rhs_packed_offset_matmul_clamp_f32_f32p2vlx1_f32p2vlx1b_2vlx2vl_sme_mopa>,+ /* .run_kernel_ex = */ &kernel_run_fn10<kai_run_matmul_clamp_f32_f32p2vlx1_f32p2vlx1b_2vlx2vl_sme_mopa>,+ },+ /* .gemm_lhs_info = */ {+ /* .get_offset = */ kai_get_lhs_offset_lhs_pack_f32p2vlx1_f32_sme,+ /* .get_packed_offset_ex = */ &lhs_offs_fn5<kai_get_lhs_packed_offset_lhs_pack_f32p2vlx1_f32_sme>,+ /* .packed_size_ex = */ &lhs_ps_fn5<kai_get_lhs_packed_size_lhs_pack_f32p2vlx1_f32_sme>,+ /* .pack_func_ex = */ &lhs_pack_void_fn9<kai_run_lhs_pack_f32p2vlx1_f32_sme>,+ },+ /* SME GEMV */+ {+ /* .get_m_step = */ kai_get_m_step_matmul_clamp_f32_f32p2vlx1_f32p2vlx1b_2vlx2vl_sme_mopa,+ /* .get_n_step = */ kai_get_n_step_matmul_clamp_f32_f32p2vlx1_f32p2vlx1b_2vlx2vl_sme_mopa,+ /* .get_mr = */ kai_get_mr_matmul_clamp_f32_f32p2vlx1_f32p2vlx1b_2vlx2vl_sme_mopa,+ /* .get_nr = */ kai_get_nr_matmul_clamp_f32_f32p2vlx1_f32p2vlx1b_2vlx2vl_sme_mopa,+ /* .get_kr = */ kai_get_kr_matmul_clamp_f32_f32p2vlx1_f32p2vlx1b_2vlx2vl_sme_mopa,+ /* .get_sr = */ kai_get_sr_matmul_clamp_f32_f32p2vlx1_f32p2vlx1b_2vlx2vl_sme_mopa,+ /* .get_dst_offset = */ kai_get_dst_offset_matmul_clamp_f32_f32p2vlx1_f32p2vlx1b_2vlx2vl_sme_mopa,+ /* .get_dst_size = */ kai_get_dst_size_matmul_clamp_f32_f32p2vlx1_f32p2vlx1b_2vlx2vl_sme_mopa,+ /* .get_lhs_offset_ex = */ nullptr,+ /* .get_rhs_packed_offset_ex = */ nullptr,+ /* .run_kernel_ex = */ nullptr,+ },+ /* .gemv_lhs_info = */ {+ /* .get_offset = */ kai_get_lhs_offset_lhs_pack_f32p2vlx1_f32_sme,+ /* .get_packed_offset_ex = */ &lhs_offs_fn5<kai_get_lhs_packed_offset_lhs_pack_f32p2vlx1_f32_sme>,+ /* .packed_size_ex = */ &lhs_ps_fn5<kai_get_lhs_packed_size_lhs_pack_f32p2vlx1_f32_sme>,+ /* .pack_func_ex = */ &lhs_pack_void_fn9<kai_run_lhs_pack_f32p2vlx1_f32_sme>,+ },+ /* .rhs_info = */ {+ /* .packed_stride = */ nullptr,+ /* .to_float = */ nullptr,+ /* .packed_size_ex = */ &rhs_ps_fn2<kai_get_rhs_packed_size_rhs_pack_nxk_f32p2vlx1biasf32_f32_f32_sme>,+ /* .packed_stride_ex = */ &rhs_stride_fn1<kai_get_rhs_packed_stride_rhs_pack_nxk_f32p2vlx1biasf32_f32_f32_sme>,+ /* .pack_func_ex = */ &rhs_pack_fn13<kai_run_rhs_pack_nxk_f32p2vlx1biasf32_f32_f32_sme>,+ },/* .required_cpu = */ CPU_FEATURE_SME,/* .lhs_type = */ GGML_TYPE_F32,/* .rhs_type = */ GGML_TYPE_F32,diff --git a/ggml/src/ggml-cpu/kleidiai/kernels.h b/ggml/src/ggml-cpu/kleidiai/kernels.hindex a46f837acb..0da5e65a0a 100644--- a/ggml/src/ggml-cpu/kleidiai/kernels.h+++ b/ggml/src/ggml-cpu/kleidiai/kernels.h@@ -11,7 +11,8 @@ enum cpu_feature {CPU_FEATURE_DOTPROD = 1,CPU_FEATURE_I8MM = 2,CPU_FEATURE_SVE = 4,- CPU_FEATURE_SME = 8+ CPU_FEATURE_SME = 8,+ CPU_FEATURE_SME2 = 16};inline cpu_feature& operator|=(cpu_feature& lhs, cpu_feature rhs) {diff --git a/ggml/src/ggml-cpu/kleidiai/kleidiai.cpp b/ggml/src/ggml-cpu/kleidiai/kleidiai.cppindex a8de7df25f..5cdcbdfe4c 100644--- a/ggml/src/ggml-cpu/kleidiai/kleidiai.cpp+++ b/ggml/src/ggml-cpu/kleidiai/kleidiai.cpp@@ -26,6 +26,9 @@#include <sys/types.h>#include <sys/stat.h>#include <unistd.h>+#ifndef HWCAP2_SME2+#define HWCAP2_SME2 (1UL << 37)+#endif#elif defined(__APPLE__)#include <string_view>#include <sys/sysctl.h>@@ -66,9 +69,15 @@ struct ggml_kleidiai_context {int chunk_multiplier;} static ctx = { CPU_FEATURE_NONE, nullptr, nullptr, nullptr, 0, -1, 4 };+static inline bool is_sme_family(cpu_feature f) {+ return (f & (CPU_FEATURE_SME | CPU_FEATURE_SME2)) != CPU_FEATURE_NONE;+}+static const char* cpu_feature_to_string(cpu_feature f) {if (f == CPU_FEATURE_NONE) {return "NONE";+ } else if ((f & CPU_FEATURE_SME2) == CPU_FEATURE_SME2) {+ return "SME2";} else if ((f & CPU_FEATURE_SME) == CPU_FEATURE_SME) {return "SME";} else if ((f & CPU_FEATURE_SVE) == CPU_FEATURE_SVE) {@@ -251,6 +260,18 @@ static void init_kleidiai_context(void) {if (sme_cores > 0) {ctx.features |= CPU_FEATURE_SME;+#if defined(__aarch64__) && defined(__linux__)+ // ARM guarantees SME2 implies SME, so only check SME2 when SME is enabled.+ if (getauxval(AT_HWCAP2) & HWCAP2_SME2) {+ ctx.features |= CPU_FEATURE_SME2;+ }+#elif defined(__aarch64__) && defined(__APPLE__)+ int feat_sme2 = 0;+ size_t size = sizeof(feat_sme2);+ if (sysctlbyname("hw.optional.arm.FEAT_SME2", &feat_sme2, &size, NULL, 0) == 0 && feat_sme2) {+ ctx.features |= CPU_FEATURE_SME2;+ }+#endif}// Kernel selection@@ -279,10 +300,13 @@ static void init_kleidiai_context(void) {ctx.sme_thread_cap = (ctx.features & CPU_FEATURE_SME) ? sme_cores : 0;if (ctx.features & CPU_FEATURE_SME) {+ const bool has_sme2 = (ctx.features & CPU_FEATURE_SME2) != CPU_FEATURE_NONE;if (sme_env_set && sme_env_ok && sme_cores > 0) {- GGML_LOG_INFO("kleidiai: SME enabled (GGML_KLEIDIAI_SME=%d override)\n", sme_cores);+ GGML_LOG_INFO("kleidiai: SME%s enabled (GGML_KLEIDIAI_SME=%d override)\n",+ has_sme2 ? "2" : "", sme_cores);} else {- GGML_LOG_INFO("kleidiai: SME enabled (runtime-detected SME cores=%d)\n", sme_cores);+ GGML_LOG_INFO("kleidiai: SME%s enabled (runtime-detected SME cores=%d)\n",+ has_sme2 ? "2" : "", sme_cores);}} else {GGML_LOG_INFO("kleidiai: SME disabled\n");@@ -442,8 +466,8 @@ static int kleidiai_collect_kernel_chain_common(return count;}- if ((primary->required_cpu & CPU_FEATURE_SME) == CPU_FEATURE_SME) {- const cpu_feature fallback_mask = static_cast<cpu_feature>(features & ~CPU_FEATURE_SME);+ if (is_sme_family(primary->required_cpu)) {+ const cpu_feature fallback_mask = static_cast<cpu_feature>(features & ~CPU_FEATURE_SME & ~CPU_FEATURE_SME2);if (fallback_mask != CPU_FEATURE_NONE) {ggml_kleidiai_kernels * fallback = select_fallback(fallback_mask);if (fallback && fallback != primary &&@@ -1054,14 +1078,14 @@ class tensor_traits : public ggml::cpu::tensor_traits {int sme_slot = -1;for (int i = 0; i < runtime_count; ++i) {- if ((runtime[i].kernels->required_cpu & CPU_FEATURE_SME) == CPU_FEATURE_SME) {+ if (is_sme_family(runtime[i].kernels->required_cpu)) {sme_slot = i;break;}}int non_sme_slot = -1;for (int i = 0; i < runtime_count; ++i) {- if ((runtime[i].kernels->required_cpu & CPU_FEATURE_SME) != CPU_FEATURE_SME) {+ if (!is_sme_family(runtime[i].kernels->required_cpu)) {non_sme_slot = i;break;}@@ -1099,7 +1123,7 @@ class tensor_traits : public ggml::cpu::tensor_traits {// Recompute SME slot based on the collapsed runtime[0]sme_slot = -1;if (runtime_count > 0 &&- (runtime[0].kernels->required_cpu & CPU_FEATURE_SME) == CPU_FEATURE_SME) {+ is_sme_family(runtime[0].kernels->required_cpu)) {sme_slot = 0;}}