LLM inference in C/C++
C++ ggml
kleidiai: Rework KleidiAI Build System/Integration (#26077)
Jonathan Clohessyd222767c7a65
diff --git a/ggml/cmake/ggml-config.cmake.in b/ggml/cmake/ggml-config.cmake.inindex abe17804a5..a28e49e834 100644--- a/ggml/cmake/ggml-config.cmake.in+++ b/ggml/cmake/ggml-config.cmake.in@@ -110,6 +110,16 @@ set_and_check(GGML_INCLUDE_DIR "@PACKAGE_GGML_INCLUDE_INSTALL_DIR@")set_and_check(GGML_LIB_DIR "@PACKAGE_GGML_LIB_INSTALL_DIR@")#set_and_check(GGML_BIN_DIR "@PACKAGE_GGML_BIN_INSTALL_DIR@")+if (NOT GGML_SHARED_LIB AND GGML_CPU_KLEIDIAI)+ unset(KLEIDIAI_LIBRARY CACHE)+ unset(KLEIDIAI_LIBRARY)+ find_library(KLEIDIAI_LIBRARY kleidiai+ REQUIRED+ HINTS ${GGML_LIB_DIR}+ NO_CMAKE_FIND_ROOT_PATH)+ list(APPEND GGML_CPU_INTERFACE_LINK_LIBRARIES ${KLEIDIAI_LIBRARY})+endif()+if(NOT TARGET ggml::ggml)find_package(Threads REQUIRED)diff --git a/ggml/src/ggml-cpu/CMakeLists.txt b/ggml/src/ggml-cpu/CMakeLists.txtindex e16ac996a4..3c6343fb2a 100644--- a/ggml/src/ggml-cpu/CMakeLists.txt+++ b/ggml/src/ggml-cpu/CMakeLists.txt@@ -576,10 +576,25 @@ function(ggml_add_cpu_backend_variant_impl tag_name)endif()if (GGML_CPU_KLEIDIAI)- message(STATUS "Using KleidiAI optimized kernels if applicable")+ # upstream repo requires at least cmake 3.16+ if (CMAKE_VERSION VERSION_LESS 3.16)+ message(FATAL_ERROR "GGML_CPU_KLEIDIAI requires CMake >= 3.16")+ endif()++ set(GGML_CPU_KLEIDIAI_AARCH64 OFF)+ if (GGML_SYSTEM_ARCH STREQUAL "ARM" AND+ (APPLE OR WIN32 OR CMAKE_SYSTEM_NAME MATCHES "^(Linux|Android)$") AND+ (CMAKE_SYSTEM_PROCESSOR MATCHES "^(aarch64|arm64|ARM64|arm64-v8a)$" OR+ CMAKE_OSX_ARCHITECTURES MATCHES "arm64" OR+ CMAKE_GENERATOR_PLATFORM_LWR STREQUAL "arm64" OR+ CMAKE_ANDROID_ARCH_ABI STREQUAL "arm64-v8a"))+ set(GGML_CPU_KLEIDIAI_AARCH64 ON)+ endif()+ if (NOT GGML_CPU_KLEIDIAI_AARCH64)+ message(FATAL_ERROR "GGML_CPU_KLEIDIAI requires a Linux, Android, Apple, or Windows AArch64/arm64 target")+ endif()- # Disable the KleidiAI tests- set(KLEIDIAI_BUILD_TESTS OFF)+ message(STATUS "Using KleidiAI optimized kernels if applicable")# Fetch KleidiAI sources:include(FetchContent)@@ -595,31 +610,49 @@ function(ggml_add_cpu_backend_variant_impl tag_name)list(APPEND KLEIDIAI_FETCH_ARGS DOWNLOAD_EXTRACT_TIMESTAMP NEW)endif()- if (CMAKE_VERSION VERSION_GREATER_EQUAL "3.28")- FetchContent_Declare(KleidiAI_Download- ${KLEIDIAI_FETCH_ARGS}- EXCLUDE_FROM_ALL- )+ FetchContent_Declare(kleidiai+ ${KLEIDIAI_FETCH_ARGS}+ )- FetchContent_MakeAvailable(KleidiAI_Download)- FetchContent_GetProperties(KleidiAI_Download SOURCE_DIR KLEIDIAI_SRC)- else()- FetchContent_Declare(KleidiAI_Download- ${KLEIDIAI_FETCH_ARGS}- )+ # Disable tests and benchmark building+ set(KLEIDIAI_BUILD_TESTS OFF CACHE BOOL "" FORCE)+ set(KLEIDIAI_BUILD_BENCHMARK OFF CACHE BOOL "" FORCE)- FetchContent_GetProperties(KleidiAI_Download+ # Use the Populate/add_subdirectory flow for compatibility with CMake 3.16.+ FetchContent_GetProperties(kleidiai+ SOURCE_DIR KLEIDIAI_SRC+ BINARY_DIR KLEIDIAI_BIN+ POPULATED KLEIDIAI_POPULATED+ )+ if (NOT KLEIDIAI_POPULATED)+ FetchContent_Populate(kleidiai)+ FetchContent_GetProperties(kleidiaiSOURCE_DIR KLEIDIAI_SRC- POPULATED KLEIDIAI_POPULATED+ BINARY_DIR KLEIDIAI_BIN)+ endif()- if (NOT KLEIDIAI_POPULATED)- FetchContent_Populate(KleidiAI_Download)- FetchContent_GetProperties(KleidiAI_Download SOURCE_DIR KLEIDIAI_SRC)+ if (NOT TARGET kleidiai)+ add_subdirectory(+ "${CMAKE_CURRENT_SOURCE_DIR}/ggml-cpu/kleidiai"+ "${CMAKE_CURRENT_BINARY_DIR}/kleidiai-wrapper"+ EXCLUDE_FROM_ALL+ )+ if (NOT CMAKE_SKIP_INSTALL_RULES AND+ (NOT DEFINED BUILD_SHARED_LIBS OR NOT BUILD_SHARED_LIBS))+ install(TARGETS kleidiai ARCHIVE)endif()endif()- add_compile_definitions(GGML_USE_CPU_KLEIDIAI)+ if (NOT TARGET kleidiai)+ message(FATAL_ERROR "KleidiAI target was not created")+ endif()++ set_target_properties(kleidiai PROPERTIES POSITION_INDEPENDENT_CODE ON)++ target_link_libraries(${GGML_CPU_NAME} PRIVATE kleidiai)++ target_compile_definitions(${GGML_CPU_NAME} PRIVATE GGML_USE_CPU_KLEIDIAI)list(APPEND GGML_CPU_SOURCESggml-cpu/kleidiai/kleidiai.cpp@@ -627,108 +660,6 @@ function(ggml_add_cpu_backend_variant_impl tag_name)ggml-cpu/kleidiai/kleidiai.hggml-cpu/kleidiai/kernels.h)-- # KleidiAI- include_directories(- ${KLEIDIAI_SRC}/- ${KLEIDIAI_SRC}/kai/- ${KLEIDIAI_SRC}/kai/ukernels/- ${KLEIDIAI_SRC}/kai/ukernels/matmul/- ${KLEIDIAI_SRC}/kai/ukernels/matmul/matmul_clamp_f32_qsi8d32p_qsi4c32p/- ${KLEIDIAI_SRC}/kai/ukernels/matmul/matmul_clamp_f32_qai8dxp_qsi8cxp/- ${KLEIDIAI_SRC}/kai/ukernels/matmul/matmul_clamp_fp32_bf16p_bf16p/- ${KLEIDIAI_SRC}/kai/ukernels/matmul/matmul_clamp_f32_f16p_qsi4c32p/- ${KLEIDIAI_SRC}/kai/ukernels/matmul/matmul_clamp_f32_f32p_f32p/- ${KLEIDIAI_SRC}/kai/ukernels/matmul/matmul_clamp_f32_f32_f32p/- ${KLEIDIAI_SRC}/kai/ukernels/matmul/pack/)-- set(ARCH_FLAGS_TEMP "${ARCH_FLAGS}")- if (NOT ARCH_FLAGS_TEMP)- string(REGEX MATCH "-march=[^ ]+" ARCH_FLAGS_TEMP "${CMAKE_C_FLAGS}")- endif()- string(FIND "${ARCH_FLAGS_TEMP}" "+dotprod" DOTPROD_ENABLED)- string(FIND "${ARCH_FLAGS_TEMP}" "+i8mm" I8MM_ENABLED)- string(FIND "${ARCH_FLAGS_TEMP}" "+sme" SME_ENABLED)- string(FIND "${ARCH_FLAGS_TEMP}" "+sve" SVE_ENABLED)-- set(PRIVATE_ARCH_FLAGS ${ARCH_FLAGS_TEMP})-- list(APPEND GGML_KLEIDIAI_SOURCES- ${KLEIDIAI_SRC}/kai/ukernels/matmul/pack/kai_lhs_quant_pack_qsi8d32p_f32.c- ${KLEIDIAI_SRC}/kai/ukernels/matmul/pack/kai_lhs_quant_pack_qsi8d32p4x8sb_f32_neon.c- ${KLEIDIAI_SRC}/kai/ukernels/matmul/pack/kai_rhs_pack_nxk_qsi4c32ps1s0scalef16_qsu4c32s16s0_neon.c- ${KLEIDIAI_SRC}/kai/ukernels/matmul/pack/kai_lhs_quant_pack_qsi8d32p_f32_neon.c- ${KLEIDIAI_SRC}/kai/ukernels/matmul/pack/kai_rhs_pack_nxk_qsi4c32pscalef16_qsu4c32s16s0.c- ${KLEIDIAI_SRC}/kai/ukernels/matmul/pack/kai_lhs_quant_pack_qai8dxp_f32.c- ${KLEIDIAI_SRC}/kai/ukernels/matmul/pack/kai_rhs_pack_nxk_qsi8cxp_qsi8cx_neon.c)-- if (NOT DOTPROD_ENABLED MATCHES -1)- list(APPEND GGML_KLEIDIAI_SOURCES- ${KLEIDIAI_SRC}/kai/ukernels/matmul/matmul_clamp_f32_qsi8d32p_qsi4c32p/kai_matmul_clamp_f32_qsi8d32p1x8_qsi4c32p4x8_1x4x32_neon_dotprod.c- ${KLEIDIAI_SRC}/kai/ukernels/matmul/matmul_clamp_f32_qsi8d32p_qsi4c32p/kai_matmul_clamp_f32_qsi8d32p1x4_qsi4c32p4x4_1x4_neon_dotprod.c- ${KLEIDIAI_SRC}/kai/ukernels/matmul/matmul_clamp_f32_qsi8d32p_qsi4c32p/kai_matmul_clamp_f32_qsi8d32p4x4_qsi4c32p4x4_16x4_neon_dotprod.c- ${KLEIDIAI_SRC}/kai/ukernels/matmul/matmul_clamp_f32_qai8dxp_qsi8cxp/kai_matmul_clamp_f32_qai8dxp4x4_qsi8cxp4x4_16x4_neon_dotprod.c- ${KLEIDIAI_SRC}/kai/ukernels/matmul/matmul_clamp_f32_qai8dxp_qsi8cxp/kai_matmul_clamp_f32_qai8dxp1x4_qsi8cxp4x4_1x4_neon_dotprod.c- ${KLEIDIAI_SRC}/kai/ukernels/matmul/matmul_clamp_f32_qai8dxp_qsi8cxp/kai_matmul_clamp_f32_qai8dxp1x8_qsi8cxp4x8_1x4_neon_dotprod.c)- endif()-- if (NOT I8MM_ENABLED MATCHES -1)- list(APPEND GGML_KLEIDIAI_SOURCES- ${KLEIDIAI_SRC}/kai/ukernels/matmul/matmul_clamp_f32_qsi8d32p_qsi4c32p/kai_matmul_clamp_f32_qsi8d32p4x8_qsi4c32p4x8_16x4_neon_i8mm.c- ${KLEIDIAI_SRC}/kai/ukernels/matmul/matmul_clamp_f32_qai8dxp_qsi8cxp/kai_matmul_clamp_f32_qai8dxp4x8_qsi8cxp4x8_16x4_neon_i8mm.c)- endif()-- if (NOT SME_ENABLED MATCHES -1)- list(APPEND GGML_KLEIDIAI_SME_SOURCES- ${KLEIDIAI_SRC}/kai/ukernels/matmul/matmul_clamp_f32_qai8dxp_qsi8cxp/kai_matmul_clamp_f32_qai8dxp1vlx4_qsi8cxp4vlx4_1vlx4vl_sme_mopa.c- ${KLEIDIAI_SRC}/kai/ukernels/matmul/matmul_clamp_f32_qai8dxp_qsi8cxp/kai_matmul_clamp_f32_qai8dxp1vlx4_qsi8cxp4vlx4_1vlx4vl_sme_mopa_asm.S- ${KLEIDIAI_SRC}/kai/ukernels/matmul/matmul_clamp_f32_qai8dxp_qsi8cxp/kai_matmul_clamp_f32_qai8dxp1x4_qsi8cxp4vlx4_1x4vl_sme_dot.c- ${KLEIDIAI_SRC}/kai/ukernels/matmul/matmul_clamp_f32_qai8dxp_qsi8cxp/kai_matmul_clamp_f32_qai8dxp1x4_qsi8cxp4vlx4_1x4vl_sme_dot_asm.S- ${KLEIDIAI_SRC}/kai/ukernels/matmul/matmul_clamp_f32_f32p_f32p/kai_matmul_clamp_f32_f32p2vlx1_f32p2vlx1b_2vlx2vl_sme_mopa.c- ${KLEIDIAI_SRC}/kai/ukernels/matmul/matmul_clamp_f32_f32p_f32p/kai_matmul_clamp_f32_f32p2vlx1_f32p2vlx1b_2vlx2vl_sme_mopa_asm.S)- set_source_files_properties(${GGML_KLEIDIAI_SME_SOURCES}- PROPERTIES COMPILE_OPTIONS "-fno-tree-vectorize;${ARCH_FLAGS_TEMP}+sve+sve2+sme")- list(APPEND GGML_CPU_SOURCES ${GGML_KLEIDIAI_SME_SOURCES})-- list(APPEND GGML_KLEIDIAI_SME2_SOURCES- ${KLEIDIAI_SRC}/kai/ukernels/matmul/matmul_clamp_f32_qsi8d32p_qsi4c32p/kai_matmul_clamp_f32_qsi8d32p1x4_qsi4c32p4vlx4_1x4vl_sme2_sdot.c- ${KLEIDIAI_SRC}/kai/ukernels/matmul/matmul_clamp_f32_qai8dxp_qsi8cxp/kai_matmul_clamp_f32_qai8dxp1vlx4_qsi8cxp4vlx4_1vlx4vl_sme2_mopa.c- ${KLEIDIAI_SRC}/kai/ukernels/matmul/matmul_clamp_f32_qai8dxp_qsi8cxp/kai_matmul_clamp_f32_qai8dxp1vlx4_qsi8cxp4vlx4_1vlx4vl_sme2_mopa_asm.S- ${KLEIDIAI_SRC}/kai/ukernels/matmul/matmul_clamp_f32_qai8dxp_qsi8cxp/kai_matmul_clamp_f32_qai8dxp1x4_qsi8cxp4vlx4_1x4vl_sme2_dot.c- ${KLEIDIAI_SRC}/kai/ukernels/matmul/matmul_clamp_f32_qai8dxp_qsi8cxp/kai_matmul_clamp_f32_qai8dxp1x4_qsi8cxp4vlx4_1x4vl_sme2_dot_asm.S- ${KLEIDIAI_SRC}/kai/ukernels/matmul/matmul_clamp_fp32_bf16p_bf16p/kai_matmul_clamp_f32_bf16p2vlx2_bf16p2vlx2_2vlx2vl_sme2_mopa.c- ${KLEIDIAI_SRC}/kai/ukernels/matmul/matmul_clamp_fp32_bf16p_bf16p/kai_matmul_clamp_f32_bf16p2vlx2_bf16p2vlx2_2vlx2vl_sme2_mopa_asm.S- ${KLEIDIAI_SRC}/kai/ukernels/matmul/matmul_clamp_f32_f16p_qsi4c32p/kai_matmul_clamp_f32_f16p1vlx2_qsi4c32p4vlx2_1vlx4vl_sme2_mopa.c- ${KLEIDIAI_SRC}/kai/ukernels/matmul/matmul_clamp_f32_f16p_qsi4c32p/kai_matmul_clamp_f32_f16p1vlx2_qsi4c32p4vlx2_1vlx4vl_sme2_mopa_asm.S- ${KLEIDIAI_SRC}/kai/ukernels/matmul/matmul_clamp_f32_f32p_f32p/kai_matmul_clamp_f32_f32p2vlx1_f32p2vlx1biasf32_sme2_mopa.c- ${KLEIDIAI_SRC}/kai/ukernels/matmul/matmul_clamp_f32_f32p_f32p/kai_matmul_clamp_f32_f32p2vlx1_f32p2vlx1biasf32_sme2_mopa_asm.S- ${KLEIDIAI_SRC}/kai/ukernels/matmul/matmul_clamp_f32_f32_f32p/kai_matmul_clamp_f32_f32_f32p2vlx1b_1x16vl_sme2_mla.c- ${KLEIDIAI_SRC}/kai/ukernels/matmul/matmul_clamp_f32_f32_f32p/kai_matmul_clamp_f32_f32_f32p2vlx1b_1x16vl_sme2_mla_asm.S- ${KLEIDIAI_SRC}/kai/ukernels/matmul/pack/kai_lhs_pack_bf16p2vlx2_f32_sme.c- ${KLEIDIAI_SRC}/kai/ukernels/matmul/pack/kai_rhs_pack_kxn_bf16p2vlx2b_f32_x32_sme.c- ${KLEIDIAI_SRC}/kai/ukernels/matmul/pack/kai_lhs_pack_f16pmrx2_f32_neon.c- ${KLEIDIAI_SRC}/kai/ukernels/matmul/pack/kai_lhs_pack_f32p2vlx1_f32_sme.c- ${KLEIDIAI_SRC}/kai/ukernels/matmul/pack/kai_lhs_pack_f32p2vlx1_f32_sme_asm.S- ${KLEIDIAI_SRC}/kai/ukernels/matmul/pack/kai_rhs_pack_nxk_f32p2vlx1biasf32_f32_f32_sme.c- ${KLEIDIAI_SRC}/kai/ukernels/matmul/pack/kai_rhs_pack_nxk_f32p2vlx1biasf32_f32_f32_sme_asm.S- ${KLEIDIAI_SRC}/kai/kai_common_sme_asm.S)- set_source_files_properties(${GGML_KLEIDIAI_SME2_SOURCES}- PROPERTIES COMPILE_OPTIONS "-fno-tree-vectorize;${ARCH_FLAGS_TEMP}+sve+sve2+sme2+fp16")- list(APPEND GGML_CPU_SOURCES ${GGML_KLEIDIAI_SME2_SOURCES})- set(PRIVATE_ARCH_FLAGS "-fno-tree-vectorize;${PRIVATE_ARCH_FLAGS}")- endif()-- if (NOT SVE_ENABLED MATCHES -1)- list(APPEND GGML_KLEIDIAI_SOURCES- ${KLEIDIAI_SRC}/kai/kai_common_sve_asm.S- ${KLEIDIAI_SRC}/kai/ukernels/matmul/matmul_clamp_f32_qsi8d32p_qsi4c32p/kai_matmul_clamp_f32_qsi8d32p1x8_qsi4c32p8x8_1x8_sve_dotprod_asm.S- ${KLEIDIAI_SRC}/kai/ukernels/matmul/matmul_clamp_f32_qsi8d32p_qsi4c32p/kai_matmul_clamp_f32_qsi8d32p1x8_qsi4c32p8x8_1x8_sve_dotprod.c- ${KLEIDIAI_SRC}/kai/ukernels/matmul/matmul_clamp_f32_qsi8d32p_qsi4c32p/kai_matmul_clamp_f32_qsi8d32p4x8_qsi4c32p8x8_16x8_sve_i8mm_asm.S- ${KLEIDIAI_SRC}/kai/ukernels/matmul/matmul_clamp_f32_qsi8d32p_qsi4c32p/kai_matmul_clamp_f32_qsi8d32p4x8_qsi4c32p8x8_16x8_sve_i8mm.c)- endif()-- set_source_files_properties(${GGML_KLEIDIAI_SOURCES} PROPERTIES COMPILE_OPTIONS "${PRIVATE_ARCH_FLAGS}")- list(APPEND GGML_CPU_SOURCES ${GGML_KLEIDIAI_SOURCES})endif()message(STATUS "Adding CPU backend variant ${GGML_CPU_NAME}: ${ARCH_FLAGS} ${ARCH_DEFINITIONS}")diff --git a/ggml/src/ggml-cpu/kleidiai/CMakeLists.txt b/ggml/src/ggml-cpu/kleidiai/CMakeLists.txtnew file mode 100644index 0000000000..b36cb6d3a9--- /dev/null+++ b/ggml/src/ggml-cpu/kleidiai/CMakeLists.txt@@ -0,0 +1,14 @@+set(BUILD_SHARED_LIBS OFF)+set(CMAKE_SKIP_INSTALL_RULES TRUE)++add_subdirectory("${KLEIDIAI_SRC}" "${KLEIDIAI_BIN}" EXCLUDE_FROM_ALL)++if (NOT TARGET kleidiai)+ message(FATAL_ERROR "KleidiAI target was not created")+endif()++if (MSVC)+ target_compile_options(kleidiai PRIVATE $<$<COMPILE_LANGUAGE:C,CXX>:/WX->)+else()+ target_compile_options(kleidiai PRIVATE $<$<COMPILE_LANGUAGE:C,CXX>:-Wno-error>)+endif()diff --git a/ggml/src/ggml-cpu/kleidiai/kernels.cpp b/ggml/src/ggml-cpu/kleidiai/kernels.cppindex 70b519f29c..d4551298f8 100644--- a/ggml/src/ggml-cpu/kleidiai/kernels.cpp+++ b/ggml/src/ggml-cpu/kleidiai/kernels.cpp@@ -3,44 +3,44 @@//// KleidiAI micro-kernels-#include "kai_matmul_clamp_f32_qsi8d32p_qsi4c32p_interface.h"-#include "kai_matmul_clamp_f32_qai8dxp_qsi8cxp_interface.h"-#include "kai_matmul_clamp_f32_qsi8d32p1x8_qsi4c32p4x8_1x4x32_neon_dotprod.h"-#include "kai_matmul_clamp_f32_qsi8d32p1x4_qsi4c32p4x4_1x4_neon_dotprod.h"-#include "kai_matmul_clamp_f32_qsi8d32p4x4_qsi4c32p4x4_16x4_neon_dotprod.h"-#include "kai_matmul_clamp_f32_qsi8d32p4x8_qsi4c32p4x8_16x4_neon_i8mm.h"-#include "kai_matmul_clamp_f32_qsi8d32p1x4_qsi4c32p4vlx4_1x4vl_sme2_sdot.h"-#include "kai_matmul_clamp_f32_bf16p2vlx2_bf16p2vlx2_2vlx2vl_sme2_mopa.h"-#include "kai_matmul_clamp_f32_qai8dxp1vlx4_qsi8cxp4vlx4_1vlx4vl_sme2_mopa.h"-#include "kai_matmul_clamp_f32_qai8dxp1x4_qsi8cxp4vlx4_1x4vl_sme2_dot.h"-#include "kai_matmul_clamp_f32_qai8dxp1vlx4_qsi8cxp4vlx4_1vlx4vl_sme_mopa.h"-#include "kai_matmul_clamp_f32_qai8dxp1x4_qsi8cxp4vlx4_1x4vl_sme_dot.h"-#include "kai_matmul_clamp_f32_qai8dxp1x8_qsi8cxp4x8_1x4_neon_dotprod.h"-#include "kai_matmul_clamp_f32_qai8dxp1x4_qsi8cxp4x4_1x4_neon_dotprod.h"-#include "kai_matmul_clamp_f32_qai8dxp4x4_qsi8cxp4x4_16x4_neon_dotprod.h"-#include "kai_matmul_clamp_f32_qai8dxp4x8_qsi8cxp4x8_16x4_neon_i8mm.h"-#include "kai_matmul_clamp_f32_qsi8d32p4x8_qsi4c32p8x8_16x8_sve_i8mm.h"-#include "kai_matmul_clamp_f32_qsi8d32p1x8_qsi4c32p8x8_1x8_sve_dotprod.h"-#include "kai_matmul_clamp_f32_f16p1vlx2_qsi4c32p4vlx2_1vlx4vl_sme2_mopa.h"-#include "kai_matmul_clamp_f32_f32p2vlx1_f32p2vlx1biasf32_sme2_mopa.h"-#include "kai_matmul_clamp_f32_f32_f32p2vlx1b_1x16vl_sme2_mla.h"-#include "kai_matmul_clamp_f32_f32p2vlx1_f32p2vlx1b_2vlx2vl_sme_mopa.h"--#include "kai_lhs_pack_bf16p2vlx2_f32_sme.h"-#include "kai_lhs_pack_f32p2vlx1_f32_sme.h"-#include "kai_lhs_quant_pack_qsi8d32p_f32.h"-#include "kai_lhs_quant_pack_qsi8d32p4x8sb_f32_neon.h"-#include "kai_lhs_quant_pack_qsi8d32p_f32_neon.h"-#include "kai_lhs_quant_pack_qai8dxp_f32.h"--#include "kai_rhs_pack_kxn_bf16p2vlx2b_f32_x32_sme.h"-#include "kai_rhs_pack_nxk_f32p2vlx1biasf32_f32_f32_sme.h"-#include "kai_rhs_pack_nxk_qsi4c32pscalef16_qsu4c32s16s0.h"-#include "kai_rhs_pack_nxk_qsi4c32ps1s0scalef16_qsu4c32s16s0_neon.h"-#include "kai_rhs_pack_nxk_qsi8cxp_qsi8cx_neon.h"-#include "kai_lhs_pack_f16pmrx2_f32_neon.h"--#include "kai_common.h"+#include "kai/ukernels/matmul/matmul_clamp_f32_qsi8d32p_qsi4c32p/kai_matmul_clamp_f32_qsi8d32p_qsi4c32p_interface.h"+#include "kai/ukernels/matmul/matmul_clamp_f32_qai8dxp_qsi8cxp/kai_matmul_clamp_f32_qai8dxp_qsi8cxp_interface.h"+#include "kai/ukernels/matmul/matmul_clamp_f32_qsi8d32p_qsi4c32p/kai_matmul_clamp_f32_qsi8d32p1x8_qsi4c32p4x8_1x4x32_neon_dotprod.h"+#include "kai/ukernels/matmul/matmul_clamp_f32_qsi8d32p_qsi4c32p/kai_matmul_clamp_f32_qsi8d32p1x4_qsi4c32p4x4_1x4_neon_dotprod.h"+#include "kai/ukernels/matmul/matmul_clamp_f32_qsi8d32p_qsi4c32p/kai_matmul_clamp_f32_qsi8d32p4x4_qsi4c32p4x4_16x4_neon_dotprod.h"+#include "kai/ukernels/matmul/matmul_clamp_f32_qsi8d32p_qsi4c32p/kai_matmul_clamp_f32_qsi8d32p4x8_qsi4c32p4x8_16x4_neon_i8mm.h"+#include "kai/ukernels/matmul/matmul_clamp_f32_qsi8d32p_qsi4c32p/kai_matmul_clamp_f32_qsi8d32p1x4_qsi4c32p4vlx4_1x4vl_sme2_sdot.h"+#include "kai/ukernels/matmul/matmul_clamp_fp32_bf16p_bf16p/kai_matmul_clamp_f32_bf16p2vlx2_bf16p2vlx2_2vlx2vl_sme2_mopa.h"+#include "kai/ukernels/matmul/matmul_clamp_f32_qai8dxp_qsi8cxp/kai_matmul_clamp_f32_qai8dxp1vlx4_qsi8cxp4vlx4_1vlx4vl_sme2_mopa.h"+#include "kai/ukernels/matmul/matmul_clamp_f32_qai8dxp_qsi8cxp/kai_matmul_clamp_f32_qai8dxp1x4_qsi8cxp4vlx4_1x4vl_sme2_dot.h"+#include "kai/ukernels/matmul/matmul_clamp_f32_qai8dxp_qsi8cxp/kai_matmul_clamp_f32_qai8dxp1vlx4_qsi8cxp4vlx4_1vlx4vl_sme_mopa.h"+#include "kai/ukernels/matmul/matmul_clamp_f32_qai8dxp_qsi8cxp/kai_matmul_clamp_f32_qai8dxp1x4_qsi8cxp4vlx4_1x4vl_sme_dot.h"+#include "kai/ukernels/matmul/matmul_clamp_f32_qai8dxp_qsi8cxp/kai_matmul_clamp_f32_qai8dxp1x8_qsi8cxp4x8_1x4_neon_dotprod.h"+#include "kai/ukernels/matmul/matmul_clamp_f32_qai8dxp_qsi8cxp/kai_matmul_clamp_f32_qai8dxp1x4_qsi8cxp4x4_1x4_neon_dotprod.h"+#include "kai/ukernels/matmul/matmul_clamp_f32_qai8dxp_qsi8cxp/kai_matmul_clamp_f32_qai8dxp4x4_qsi8cxp4x4_16x4_neon_dotprod.h"+#include "kai/ukernels/matmul/matmul_clamp_f32_qai8dxp_qsi8cxp/kai_matmul_clamp_f32_qai8dxp4x8_qsi8cxp4x8_16x4_neon_i8mm.h"+#include "kai/ukernels/matmul/matmul_clamp_f32_qsi8d32p_qsi4c32p/kai_matmul_clamp_f32_qsi8d32p4x8_qsi4c32p8x8_16x8_sve_i8mm.h"+#include "kai/ukernels/matmul/matmul_clamp_f32_qsi8d32p_qsi4c32p/kai_matmul_clamp_f32_qsi8d32p1x8_qsi4c32p8x8_1x8_sve_dotprod.h"+#include "kai/ukernels/matmul/matmul_clamp_f32_f16p_qsi4c32p/kai_matmul_clamp_f32_f16p1vlx2_qsi4c32p4vlx2_1vlx4vl_sme2_mopa.h"+#include "kai/ukernels/matmul/matmul_clamp_f32_f32p_f32p/kai_matmul_clamp_f32_f32p2vlx1_f32p2vlx1biasf32_sme2_mopa.h"+#include "kai/ukernels/matmul/matmul_clamp_f32_f32_f32p/kai_matmul_clamp_f32_f32_f32p2vlx1b_1x16vl_sme2_mla.h"+#include "kai/ukernels/matmul/matmul_clamp_f32_f32p_f32p/kai_matmul_clamp_f32_f32p2vlx1_f32p2vlx1b_2vlx2vl_sme_mopa.h"++#include "kai/ukernels/matmul/pack/kai_lhs_pack_bf16p2vlx2_f32_sme.h"+#include "kai/ukernels/matmul/pack/kai_lhs_pack_f32p2vlx1_f32_sme.h"+#include "kai/ukernels/matmul/pack/kai_lhs_quant_pack_qsi8d32p_f32.h"+#include "kai/ukernels/matmul/pack/kai_lhs_quant_pack_qsi8d32p4x8sb_f32_neon.h"+#include "kai/ukernels/matmul/pack/kai_lhs_quant_pack_qsi8d32p_f32_neon.h"+#include "kai/ukernels/matmul/pack/kai_lhs_quant_pack_qai8dxp_f32.h"++#include "kai/ukernels/matmul/pack/kai_rhs_pack_kxn_bf16p2vlx2b_f32_x32_sme.h"+#include "kai/ukernels/matmul/pack/kai_rhs_pack_nxk_f32p2vlx1biasf32_f32_f32_sme.h"+#include "kai/ukernels/matmul/pack/kai_rhs_pack_nxk_qsi4c32pscalef16_qsu4c32s16s0.h"+#include "kai/ukernels/matmul/pack/kai_rhs_pack_nxk_qsi4c32ps1s0scalef16_qsu4c32s16s0_neon.h"+#include "kai/ukernels/matmul/pack/kai_rhs_pack_nxk_qsi8cxp_qsi8cx_neon.h"+#include "kai/ukernels/matmul/pack/kai_lhs_pack_f16pmrx2_f32_neon.h"++#include "kai/kai_common.h"#include "simd-mappings.h"@@ -328,9 +328,8 @@ static void dequantize_row_qsi8cxp(}static ggml_kleidiai_kernels gemm_gemv_kernels[] = {-#if defined(__ARM_FEATURE_SME){- /* SME GEMM */+ /* SME2 GEMM *//* .kern_info = */ {/* .get_m_step = */ kai_get_m_step_matmul_clamp_f32_f16p1vlx2_qsi4c32p4vlx2_1vlx4vl_sme2_mopa,/* .get_n_step = */ kai_get_n_step_matmul_clamp_f32_f16p1vlx2_qsi4c32p4vlx2_1vlx4vl_sme2_mopa,@@ -351,7 +350,7 @@ static ggml_kleidiai_kernels gemm_gemv_kernels[] = {/* .packed_size_ex = */ &lhs_ps_fn6<kai_get_lhs_packed_size_lhs_pack_f16pmrx2_f32_neon>,/* .pack_func_ex = */ &lhs_pack_void_fn10<kai_run_lhs_pack_f16pmrx2_f32_neon>,},- /* SME GEMV */+ /* SME2 GEMV *//* .kern_info = */ {/* .get_m_step = */ kai_get_m_step_matmul_clamp_f32_qsi8d32p1x4_qsi4c32p4vlx4_1x4vl_sme2_sdot,/* .get_n_step = */ kai_get_n_step_matmul_clamp_f32_qsi8d32p1x4_qsi4c32p4vlx4_1x4vl_sme2_sdot,@@ -378,13 +377,13 @@ static ggml_kleidiai_kernels gemm_gemv_kernels[] = {/* .packed_stride_ex = */ &rhs_stride_fn4<kai_get_rhs_packed_stride_rhs_pack_nxk_qsi4c32ps1s0scalef16_qsu4c32s16s0_neon>,/* .pack_func_ex = */ &rhs_pack_fn12<kai_run_rhs_pack_nxk_qsi4c32ps1s0scalef16_qsu4c32s16s0_neon>,},- /* .required_cpu = */ CPU_FEATURE_SME2,+ /* .required_cpu = */ CPU_FEATURE_SME2 | CPU_FEATURE_FP16,/* .lhs_type = */ GGML_TYPE_F32,/* .rhs_type = */ GGML_TYPE_Q4_0,/* .op_type = */ GGML_TYPE_F32,},{- /* SME GEMM */+ /* SME2 GEMM *//* .kern_info = */ {/* .get_m_step = */ kai_get_m_step_matmul_clamp_f32_bf16p2vlx2_bf16p2vlx2_2vlx2vl_sme2_mopa,/* .get_n_step = */ kai_get_n_step_matmul_clamp_f32_bf16p2vlx2_bf16p2vlx2_2vlx2vl_sme2_mopa,@@ -404,7 +403,7 @@ static ggml_kleidiai_kernels gemm_gemv_kernels[] = {/* .packed_size_ex = */ &lhs_ps_fn5<kai_get_lhs_packed_size_lhs_pack_bf16p2vlx2_f32_sme>,/* .pack_func_ex = */ &lhs_pack_void_fn9<kai_run_lhs_pack_bf16p2vlx2_f32_sme>,},- /* SME GEMV */+ /* SME2 GEMV *//* .kern_info = */ {/* .get_m_step = */ kai_get_m_step_matmul_clamp_f32_bf16p2vlx2_bf16p2vlx2_2vlx2vl_sme2_mopa,/* .get_n_step = */ kai_get_n_step_matmul_clamp_f32_bf16p2vlx2_bf16p2vlx2_2vlx2vl_sme2_mopa,@@ -436,9 +435,7 @@ static ggml_kleidiai_kernels gemm_gemv_kernels[] = {/* .rhs_type = */ GGML_TYPE_F16,/* .op_type = */ GGML_TYPE_F32,},-#endif#if defined(__APPLE__)-#if defined(__ARM_FEATURE_DOTPROD){/* DOTPROD GEMM *//* .kern_info = */ {@@ -492,8 +489,6 @@ static ggml_kleidiai_kernels gemm_gemv_kernels[] = {/* .rhs_type = */ GGML_TYPE_Q4_0,/* .op_type = */ GGML_TYPE_F32,},-#endif-#if defined(__ARM_FEATURE_MATMUL_INT8){/* i8mm GEMM *//* .kern_info = */ {@@ -515,7 +510,7 @@ static ggml_kleidiai_kernels gemm_gemv_kernels[] = {/* .packed_size_ex = */ &lhs_ps_fn6<kai_get_lhs_packed_size_lhs_quant_pack_qsi8d32p4x8sb_f32_neon>,/* .pack_func_ex = */ &lhs_pack_float_fn10<kai_run_lhs_quant_pack_qsi8d32p4x8sb_f32_neon>,},- /* i8mm GEMV */+ /* DOTPROD GEMV *//* .kern_info = */ {/* .get_m_step = */ kai_get_m_step_matmul_clamp_f32_qsi8d32p1x8_qsi4c32p4x8_1x4x32_neon_dotprod,/* .get_n_step = */ kai_get_n_step_matmul_clamp_f32_qsi8d32p1x8_qsi4c32p4x8_1x4x32_neon_dotprod,@@ -542,14 +537,12 @@ static ggml_kleidiai_kernels gemm_gemv_kernels[] = {/* .packed_stride_ex = */ &rhs_stride_fn4<kai_get_rhs_packed_stride_rhs_pack_nxk_qsi4c32pscalef16_qsu4c32s16s0>,/* .pack_func_ex = */ &rhs_pack_fn12<kai_run_rhs_pack_nxk_qsi4c32pscalef16_qsu4c32s16s0>,},- /* .required_cpu = */ CPU_FEATURE_I8MM,+ /* .required_cpu = */ CPU_FEATURE_I8MM | CPU_FEATURE_DOTPROD,/* .lhs_type = */ GGML_TYPE_F32,/* .rhs_type = */ GGML_TYPE_Q4_0,/* .op_type = */ GGML_TYPE_F32,},-#endif#else-#if defined(__ARM_FEATURE_SVE){/* SVE i8mm GEMM *//* .kern_info = */ {@@ -603,8 +596,6 @@ static ggml_kleidiai_kernels gemm_gemv_kernels[] = {/* .rhs_type = */ GGML_TYPE_Q4_0,/* .op_type = */ GGML_TYPE_F32,},-#endif-#if defined(__ARM_FEATURE_MATMUL_INT8){/* i8mm GEMM *//* .kern_info = */ {@@ -626,7 +617,7 @@ static ggml_kleidiai_kernels gemm_gemv_kernels[] = {/* .packed_size_ex = */ &lhs_ps_fn6<kai_get_lhs_packed_size_lhs_quant_pack_qsi8d32p4x8sb_f32_neon>,/* .pack_func_ex = */ &lhs_pack_float_fn10<kai_run_lhs_quant_pack_qsi8d32p4x8sb_f32_neon>,},- /* i8mm GEMV */+ /* DOTPROD GEMV *//* .kern_info = */ {/* .get_m_step = */ kai_get_m_step_matmul_clamp_f32_qsi8d32p1x8_qsi4c32p4x8_1x4x32_neon_dotprod,/* .get_n_step = */ kai_get_n_step_matmul_clamp_f32_qsi8d32p1x8_qsi4c32p4x8_1x4x32_neon_dotprod,@@ -653,13 +644,11 @@ static ggml_kleidiai_kernels gemm_gemv_kernels[] = {/* .packed_stride_ex = */ &rhs_stride_fn4<kai_get_rhs_packed_stride_rhs_pack_nxk_qsi4c32pscalef16_qsu4c32s16s0>,/* .pack_func_ex = */ &rhs_pack_fn12<kai_run_rhs_pack_nxk_qsi4c32pscalef16_qsu4c32s16s0>,},- /* .required_cpu = */ CPU_FEATURE_I8MM,+ /* .required_cpu = */ CPU_FEATURE_I8MM | CPU_FEATURE_DOTPROD,/* .lhs_type = */ GGML_TYPE_F32,/* .rhs_type = */ GGML_TYPE_Q4_0,/* .op_type = */ GGML_TYPE_F32,},-#endif // __ARM_FEATURE_MATMUL_INT8-#if defined(__ARM_FEATURE_DOTPROD){/* DOTPROD GEMM *//* .kern_info = */ {@@ -713,15 +702,13 @@ static ggml_kleidiai_kernels gemm_gemv_kernels[] = {/* .rhs_type = */ GGML_TYPE_Q4_0,/* .op_type = */ GGML_TYPE_F32,},-#endif#endif{ /* Sentinel */ }};static ggml_kleidiai_kernels gemm_gemv_kernels_q8[] = {-#if defined(__ARM_FEATURE_SME){- /* SME GEMM */+ /* SME2 GEMM */{/* .get_m_step = */ kai_get_m_step_matmul_clamp_f32_qai8dxp1vlx4_qsi8cxp4vlx4_1vlx4vl_sme2_mopa,/* .get_n_step = */ kai_get_n_step_matmul_clamp_f32_qai8dxp1vlx4_qsi8cxp4vlx4_1vlx4vl_sme2_mopa,@@ -741,7 +728,7 @@ static ggml_kleidiai_kernels gemm_gemv_kernels_q8[] = {/* .packed_size_ex = */ &lhs_ps_fn5<kai_get_lhs_packed_size_lhs_quant_pack_qai8dxp_f32>,/* .pack_func_ex = */ &lhs_pack_float_fn9_no_bl<kai_run_lhs_quant_pack_qai8dxp_f32>,},- /* SME GEMV */+ /* SME2 GEMV */{/* .get_m_step = */ kai_get_m_step_matmul_clamp_f32_qai8dxp1x4_qsi8cxp4vlx4_1x4vl_sme2_dot,/* .get_n_step = */ kai_get_n_step_matmul_clamp_f32_qai8dxp1x4_qsi8cxp4vlx4_1x4vl_sme2_dot,@@ -826,8 +813,6 @@ static ggml_kleidiai_kernels gemm_gemv_kernels_q8[] = {/* .rhs_type = */ GGML_TYPE_Q8_0,/* .op_type = */ GGML_TYPE_F32,},-#endif-#if defined(__ARM_FEATURE_MATMUL_INT8){/* I8MM GEMM */{@@ -876,13 +861,11 @@ static ggml_kleidiai_kernels gemm_gemv_kernels_q8[] = {/* .packed_stride_ex = */ &rhs_stride_fn4<kai_get_rhs_packed_stride_rhs_pack_nxk_qsi8cxp_qsi8cx_neon>,/* .pack_func_ex = */ &rhs_pack_scale_fn12<kai_run_rhs_pack_nxk_qsi8cxp_qsi8cx_neon>,},- /* .required_cpu = */ CPU_FEATURE_I8MM,+ /* .required_cpu = */ CPU_FEATURE_I8MM | CPU_FEATURE_DOTPROD,/* .lhs_type = */ GGML_TYPE_F32,/* .rhs_type = */ GGML_TYPE_Q8_0,/* .op_type = */ GGML_TYPE_F32,},-#endif-#if defined(__ARM_FEATURE_DOTPROD){/* DOTPROD GEMM */{@@ -936,12 +919,10 @@ static ggml_kleidiai_kernels gemm_gemv_kernels_q8[] = {/* .rhs_type = */ GGML_TYPE_Q8_0,/* .op_type = */ GGML_TYPE_F32,},-#endif{ /* Sentinel */ }};static ggml_kleidiai_kernels ggml_kleidiai_kernels_f32[] = {-#if defined(__ARM_FEATURE_SME){/* SME2 GEMM */{@@ -1048,7 +1029,6 @@ static ggml_kleidiai_kernels ggml_kleidiai_kernels_f32[] = {/* .rhs_type = */ GGML_TYPE_F32,/* .op_type = */ GGML_TYPE_F32,},-#endif{ /* Sentinel */ }};@@ -1056,10 +1036,6 @@ ggml_kleidiai_kernels * ggml_kleidiai_select_kernels(cpu_feature cpu_features, cggml_kleidiai_kernels * kernel = nullptr;if (tensor->op == GGML_OP_MUL_MAT && tensor->src[0] != nullptr && tensor->src[1] != nullptr) {-#if defined(__ARM_FEATURE_SME) || \- defined(__ARM_FEATURE_DOTPROD) || \- defined(__ARM_FEATURE_MATMUL_INT8) || \- defined(__ARM_FEATURE_SVE)auto try_table = [&](auto & table) {for (size_t i = 0; i < NELEMS(table) - 1; ++i) {if ((cpu_features & table[i].required_cpu) == table[i].required_cpu &&@@ -1080,12 +1056,6 @@ ggml_kleidiai_kernels * ggml_kleidiai_select_kernels(cpu_feature cpu_features, c} else {try_table(gemm_gemv_kernels);}-#else- GGML_UNUSED(gemm_gemv_kernels);- GGML_UNUSED(gemm_gemv_kernels_q8);- GGML_UNUSED(ggml_kleidiai_kernels_f32);- GGML_UNUSED(cpu_features);-#endif}return kernel;@@ -1094,19 +1064,13 @@ ggml_kleidiai_kernels * ggml_kleidiai_select_kernels(cpu_feature cpu_features, cggml_kleidiai_kernels * ggml_kleidiai_select_kernels_q4_0(cpu_feature features) {ggml_kleidiai_kernels * kernels = nullptr;-#if defined(__ARM_FEATURE_SME) || \- defined(__ARM_FEATURE_DOTPROD) || \- defined(__ARM_FEATURE_MATMUL_INT8) || \- defined(__ARM_FEATURE_SVE)for (size_t i = 0; i < NELEMS(gemm_gemv_kernels) - 1; ++i) {- if ((features & gemm_gemv_kernels[i].required_cpu) == gemm_gemv_kernels[i].required_cpu) {+ if ((features & gemm_gemv_kernels[i].required_cpu) == gemm_gemv_kernels[i].required_cpu &&+ gemm_gemv_kernels[i].rhs_type == GGML_TYPE_Q4_0) {kernels = &gemm_gemv_kernels[i];break;}}-#else- GGML_UNUSED(features);-#endifreturn kernels;}@@ -1114,16 +1078,12 @@ ggml_kleidiai_kernels * ggml_kleidiai_select_kernels_q4_0(cpu_feature features)ggml_kleidiai_kernels * ggml_kleidiai_select_kernels_q8_0(cpu_feature features) {ggml_kleidiai_kernels * kernels = nullptr;-#if defined(__ARM_FEATURE_SME) || defined(__ARM_FEATURE_DOTPROD) || defined(__ARM_FEATURE_MATMUL_INT8)for (size_t i = 0; i < NELEMS(gemm_gemv_kernels_q8) - 1; ++i) {if ((features & gemm_gemv_kernels_q8[i].required_cpu) == gemm_gemv_kernels_q8[i].required_cpu) {kernels = &gemm_gemv_kernels_q8[i];break;}}-#else- GGML_UNUSED(features);-#endifreturn kernels;}@@ -1131,16 +1091,11 @@ ggml_kleidiai_kernels * ggml_kleidiai_select_kernels_q8_0(cpu_feature features)ggml_kleidiai_kernels * ggml_kleidiai_select_kernels_f32(cpu_feature features) {ggml_kleidiai_kernels * kernels = nullptr;-#if defined(__ARM_FEATURE_SME)for (size_t i = 0; i < NELEMS(ggml_kleidiai_kernels_f32) - 1; ++i) {if ((features & ggml_kleidiai_kernels_f32[i].required_cpu) == ggml_kleidiai_kernels_f32[i].required_cpu) {kernels = &ggml_kleidiai_kernels_f32[i];break;}}-#else- GGML_UNUSED(features);-#endif-return kernels;}diff --git a/ggml/src/ggml-cpu/kleidiai/kernels.h b/ggml/src/ggml-cpu/kleidiai/kernels.hindex 0da5e65a0a..1da8610eae 100644--- a/ggml/src/ggml-cpu/kleidiai/kernels.h+++ b/ggml/src/ggml-cpu/kleidiai/kernels.h@@ -1,4 +1,4 @@-// SPDX-FileCopyrightText: Copyright 2025 Arm Limited and/or its affiliates <[email protected]>+// SPDX-FileCopyrightText: Copyright 2025-2026 Arm Limited and/or its affiliates <[email protected]>// SPDX-License-Identifier: MIT//@@ -12,7 +12,8 @@ enum cpu_feature {CPU_FEATURE_I8MM = 2,CPU_FEATURE_SVE = 4,CPU_FEATURE_SME = 8,- CPU_FEATURE_SME2 = 16+ CPU_FEATURE_SME2 = 16,+ CPU_FEATURE_FP16 = 32};inline cpu_feature& operator|=(cpu_feature& lhs, cpu_feature rhs) {diff --git a/ggml/src/ggml-cpu/kleidiai/kleidiai.cpp b/ggml/src/ggml-cpu/kleidiai/kleidiai.cppindex 6729ae8422..92d7fd644f 100644--- a/ggml/src/ggml-cpu/kleidiai/kleidiai.cpp+++ b/ggml/src/ggml-cpu/kleidiai/kleidiai.cpp@@ -48,7 +48,7 @@#include "kernels.h"-#include "kai_common.h"+#include "kai/kai_common.h"#define GGML_COMMON_DECL_CPP#include "ggml-common.h"@@ -316,6 +316,7 @@ static void init_kleidiai_context(void) {ctx.features = (runtime_feat.has_dotprod ? CPU_FEATURE_DOTPROD : CPU_FEATURE_NONE) |(runtime_feat.has_i8mm ? CPU_FEATURE_I8MM : CPU_FEATURE_NONE) |+ (runtime_feat.has_fp16 ? CPU_FEATURE_FP16 : CPU_FEATURE_NONE) |(runtime_feat.sve_cnt == QK8_0 ? CPU_FEATURE_SVE : CPU_FEATURE_NONE);if (env_threads) {