LLM inference in C/C++
C++ ggml
metal : add metallib build support for xcframework (#28163)
Jhen-Jie Hongb96806d96061
diff --git a/build-xcframework.sh b/build-xcframework.shindex e405a1c0f6..e2a2684cc1 100755--- a/build-xcframework.sh+++ b/build-xcframework.sh@@ -18,7 +18,7 @@ LLAMA_BUILD_TESTS=OFFLLAMA_BUILD_SERVER=OFFLLAMA_BUILD_MTMD=ONGGML_METAL=ON-GGML_METAL_EMBED_LIBRARY=ON+GGML_METAL_EMBED_LIBRARY=${GGML_METAL_EMBED_LIBRARY:-ON}GGML_BLAS_DEFAULT=ONGGML_OPENMP=OFF@@ -169,6 +169,14 @@ setup_framework_structure() {cp tools/mtmd/mtmd.h ${header_path}cp tools/mtmd/mtmd-helper.h ${header_path}+ if [[ "$GGML_METAL_EMBED_LIBRARY" == "OFF" ]]; then+ if [[ "$platform" == "macos" ]]; then+ cp ${build_dir}/bin/*.metallib ${build_dir}/framework/${framework_name}.framework/Versions/A/Resources/+ else+ cp ${build_dir}/bin/*.metallib ${build_dir}/framework/${framework_name}.framework/+ fi+ fi+# Create module map (common for all platforms)cat > ${module_path}module.modulemap << EOFframework module llama {@@ -450,6 +458,7 @@ build_ios_sim() {-DIOS=ON \-DCMAKE_SYSTEM_NAME=iOS \-DCMAKE_OSX_SYSROOT=iphonesimulator \+ -DGGML_METAL_TARGET_OS=ios \-DCMAKE_OSX_ARCHITECTURES="arm64;x86_64" \-DCMAKE_XCODE_ATTRIBUTE_SUPPORTED_PLATFORMS=iphonesimulator \-DCMAKE_C_FLAGS="${COMMON_C_FLAGS}" \@@ -467,6 +476,7 @@ build_ios_device() {-DCMAKE_OSX_DEPLOYMENT_TARGET=${IOS_MIN_OS_VERSION} \-DCMAKE_SYSTEM_NAME=iOS \-DCMAKE_OSX_SYSROOT=iphoneos \+ -DGGML_METAL_TARGET_OS=ios \-DCMAKE_OSX_ARCHITECTURES="arm64" \-DCMAKE_XCODE_ATTRIBUTE_SUPPORTED_PLATFORMS=iphoneos \-DCMAKE_C_FLAGS="${COMMON_C_FLAGS}" \@@ -498,6 +508,7 @@ build_visionos() {-DCMAKE_OSX_ARCHITECTURES="arm64" \-DCMAKE_SYSTEM_NAME=visionOS \-DCMAKE_OSX_SYSROOT=xros \+ -DGGML_METAL_TARGET_OS=xros \-DCMAKE_XCODE_ATTRIBUTE_SUPPORTED_PLATFORMS=xros \-DCMAKE_C_FLAGS="${COMMON_C_FLAGS}" \-DCMAKE_CXX_FLAGS="${COMMON_CXX_FLAGS}" \@@ -516,6 +527,7 @@ build_visionos_sim() {-DCMAKE_OSX_ARCHITECTURES="arm64;x86_64" \-DCMAKE_SYSTEM_NAME=visionOS \-DCMAKE_OSX_SYSROOT=xrsimulator \+ -DGGML_METAL_TARGET_OS=xros \-DCMAKE_XCODE_ATTRIBUTE_SUPPORTED_PLATFORMS=xrsimulator \-DCMAKE_C_FLAGS="${COMMON_C_FLAGS}" \-DCMAKE_CXX_FLAGS="${COMMON_CXX_FLAGS}" \@@ -534,6 +546,7 @@ build_tvos_sim() {-DCMAKE_OSX_DEPLOYMENT_TARGET=${TVOS_MIN_OS_VERSION} \-DCMAKE_SYSTEM_NAME=tvOS \-DCMAKE_OSX_SYSROOT=appletvsimulator \+ -DGGML_METAL_TARGET_OS=tvos \-DCMAKE_OSX_ARCHITECTURES="arm64;x86_64" \-DGGML_METAL=ON \-DCMAKE_XCODE_ATTRIBUTE_SUPPORTED_PLATFORMS=appletvsimulator \@@ -552,6 +565,7 @@ build_tvos_device() {-DCMAKE_OSX_DEPLOYMENT_TARGET=${TVOS_MIN_OS_VERSION} \-DCMAKE_SYSTEM_NAME=tvOS \-DCMAKE_OSX_SYSROOT=appletvos \+ -DGGML_METAL_TARGET_OS=tvos \-DCMAKE_OSX_ARCHITECTURES="arm64" \-DGGML_METAL=ON \-DCMAKE_XCODE_ATTRIBUTE_SUPPORTED_PLATFORMS=appletvos \diff --git a/ggml/CMakeLists.txt b/ggml/CMakeLists.txtindex c4a8450d1c..0ac2b15c48 100644--- a/ggml/CMakeLists.txt+++ b/ggml/CMakeLists.txt@@ -242,6 +242,8 @@ option(GGML_METAL_EMBED_LIBRARY "ggml: embed Metal library"set (GGML_METAL_MACOSX_VERSION_MIN "" CACHE STRING"ggml: metal minimum macOS version")set (GGML_METAL_STD "" CACHE STRING "ggml: metal standard version (-std flag)")+set (GGML_METAL_TARGET_OS "macos" CACHE STRING+ "ggml: metal -mtargetos OS name (macos, ios, xros, tvos)")option(GGML_OPENMP "ggml: use OpenMP" ON)option(GGML_OPENMP_FETCH "ggml: fetch LLVM OpenMP" OFF)option(GGML_RPC "ggml: use RPC" OFF)diff --git a/ggml/src/ggml-metal/CMakeLists.txt b/ggml/src/ggml-metal/CMakeLists.txtindex 2094a409f9..a661e710a2 100644--- a/ggml/src/ggml-metal/CMakeLists.txt+++ b/ggml/src/ggml-metal/CMakeLists.txt@@ -127,6 +127,18 @@ else()configure_file(${src} ${CMAKE_RUNTIME_OUTPUT_DIRECTORY}/${src} COPYONLY)endforeach()+ # CMAKE_OSX_SYSROOT is an SDK name or path - xcrun accepts both+ set(METAL_SDK ${CMAKE_OSX_SYSROOT})+ if (NOT METAL_SDK)+ set(METAL_SDK macosx)+ endif()++ if (CMAKE_OSX_SYSROOT MATCHES "[Ss]imulator")+ set(METAL_TARGET_SIM "-simulator")+ else()+ set(METAL_TARGET_SIM "")+ endif()+if (GGML_METAL_SHADER_DEBUG)# note: disabling fast math is needed in order to pass tests/test-backend-ops# note: adding -fno-inline fixes the tests when using MTL_SHADER_VALIDATION=1@@ -138,9 +150,19 @@ else()set(XC_FLAGS -O3)endif()+ execute_process(COMMAND xcrun -sdk ${METAL_SDK} --show-sdk-version OUTPUT_VARIABLE METAL_SDK_VERSION OUTPUT_STRIP_TRAILING_WHITESPACE)+ if (METAL_SDK_VERSION VERSION_GREATER_EQUAL 26.0)+ set(GGML_METAL_HAS_TENSOR_LIB ON)+ else()+ message(STATUS "Metal SDK ${METAL_SDK_VERSION} does not support the tensor API, skipping ggml-tensor.metallib")+ endif()+if (GGML_METAL_MACOSX_VERSION_MIN)message(STATUS "Adding -mmacosx-version-min=${GGML_METAL_MACOSX_VERSION_MIN} flag to metal compilation")list (APPEND XC_FLAGS -mmacosx-version-min=${GGML_METAL_MACOSX_VERSION_MIN})+ elseif (NOT GGML_METAL_TARGET_OS STREQUAL "macos" AND CMAKE_OSX_DEPLOYMENT_TARGET)+ message(STATUS "Adding -mtargetos=${GGML_METAL_TARGET_OS}${CMAKE_OSX_DEPLOYMENT_TARGET}${METAL_TARGET_SIM} flag to metal compilation")+ list (APPEND XC_FLAGS -mtargetos=${GGML_METAL_TARGET_OS}${CMAKE_OSX_DEPLOYMENT_TARGET}${METAL_TARGET_SIM})endif()if (GGML_METAL_STD)@@ -156,33 +178,41 @@ else()list(APPEND AIR_FILES ${AIR})add_custom_command(OUTPUT ${AIR}- COMMAND xcrun -sdk macosx metal ${XC_FLAGS} -I ${CMAKE_RUNTIME_OUTPUT_DIRECTORY} -c ${CMAKE_RUNTIME_OUTPUT_DIRECTORY}/${src} -o ${AIR}+ COMMAND xcrun -sdk ${METAL_SDK} metal ${XC_FLAGS} -I ${CMAKE_RUNTIME_OUTPUT_DIRECTORY} -c ${CMAKE_RUNTIME_OUTPUT_DIRECTORY}/${src} -o ${AIR}DEPENDS ${src} kernels/common.h kernels/dequantize.h kernels/quantize.h ${METALLIB_COMMON} ggml-metal-impl.hCOMMENT "Compiling ${src}"VERBATIM)endforeach()+ set(METALLIB_FILES ${CMAKE_RUNTIME_OUTPUT_DIRECTORY}/default.metallib)+# the tensor API kernels go in a separate metallib, loaded only where supported- set(AIR_MM_TENSOR "${CMAKE_RUNTIME_OUTPUT_DIRECTORY}/mul_mm_tensor.air")- add_custom_command(- OUTPUT ${AIR_MM_TENSOR}- COMMAND xcrun -sdk macosx metal ${XC_FLAGS} -DGGML_METAL_HAS_TENSOR -I ${CMAKE_RUNTIME_OUTPUT_DIRECTORY} -c ${CMAKE_RUNTIME_OUTPUT_DIRECTORY}/kernels/mul_mm.metal -o ${AIR_MM_TENSOR}- DEPENDS kernels/mul_mm.metal kernels/common.h kernels/dequantize.h ${METALLIB_COMMON} ggml-metal-impl.h- COMMENT "Compiling kernels/mul_mm.metal (tensor API)"- VERBATIM- )+ if (GGML_METAL_HAS_TENSOR_LIB)+ set(AIR_MM_TENSOR "${CMAKE_RUNTIME_OUTPUT_DIRECTORY}/mul_mm_tensor.air")+ # the tensor API needs OS 26++ set(XC_FLAGS_TENSOR ${XC_FLAGS} -mtargetos=${GGML_METAL_TARGET_OS}26.0${METAL_TARGET_SIM})+ add_custom_command(+ OUTPUT ${AIR_MM_TENSOR}+ COMMAND xcrun -sdk ${METAL_SDK} metal ${XC_FLAGS_TENSOR} -DGGML_METAL_HAS_TENSOR -I ${CMAKE_RUNTIME_OUTPUT_DIRECTORY} -c ${CMAKE_RUNTIME_OUTPUT_DIRECTORY}/kernels/mul_mm.metal -o ${AIR_MM_TENSOR}+ DEPENDS kernels/mul_mm.metal kernels/common.h kernels/dequantize.h ${METALLIB_COMMON} ggml-metal-impl.h+ COMMENT "Compiling kernels/mul_mm.metal (tensor API)"+ VERBATIM+ )- add_custom_command(- OUTPUT ${CMAKE_RUNTIME_OUTPUT_DIRECTORY}/ggml-tensor.metallib- COMMAND xcrun -sdk macosx metallib ${AIR_MM_TENSOR} -o ${CMAKE_RUNTIME_OUTPUT_DIRECTORY}/ggml-tensor.metallib- DEPENDS ${AIR_MM_TENSOR}- COMMENT "Linking tensor API Metal kernels into ggml-tensor.metallib"- )+ add_custom_command(+ OUTPUT ${CMAKE_RUNTIME_OUTPUT_DIRECTORY}/ggml-tensor.metallib+ COMMAND xcrun -sdk ${METAL_SDK} metallib ${AIR_MM_TENSOR} -o ${CMAKE_RUNTIME_OUTPUT_DIRECTORY}/ggml-tensor.metallib+ DEPENDS ${AIR_MM_TENSOR}+ COMMENT "Linking tensor API Metal kernels into ggml-tensor.metallib"+ )++ list(APPEND METALLIB_FILES ${CMAKE_RUNTIME_OUTPUT_DIRECTORY}/ggml-tensor.metallib)+ endif()add_custom_command(OUTPUT ${CMAKE_RUNTIME_OUTPUT_DIRECTORY}/default.metallib- COMMAND xcrun -sdk macosx metallib ${AIR_FILES} -o ${CMAKE_RUNTIME_OUTPUT_DIRECTORY}/default.metallib+ COMMAND xcrun -sdk ${METAL_SDK} metallib ${AIR_FILES} -o ${CMAKE_RUNTIME_OUTPUT_DIRECTORY}/default.metallibCOMMAND rm -f ${CMAKE_RUNTIME_OUTPUT_DIRECTORY}/ggml-common.hCOMMAND rm -f ${CMAKE_RUNTIME_OUTPUT_DIRECTORY}/ggml-metal-impl.hCOMMAND rm -rf ${CMAKE_RUNTIME_OUTPUT_DIRECTORY}/kernels@@ -192,8 +222,7 @@ else()add_custom_target(ggml-metal-lib ALL- DEPENDS ${CMAKE_RUNTIME_OUTPUT_DIRECTORY}/default.metallib- ${CMAKE_RUNTIME_OUTPUT_DIRECTORY}/ggml-tensor.metallib+ DEPENDS ${METALLIB_FILES})endif() # GGML_METAL_EMBED_LIBRARY@@ -205,8 +234,7 @@ if (NOT GGML_METAL_EMBED_LIBRARY))install(- FILES ${CMAKE_RUNTIME_OUTPUT_DIRECTORY}/default.metallib- ${CMAKE_RUNTIME_OUTPUT_DIRECTORY}/ggml-tensor.metallib+ FILES ${METALLIB_FILES}DESTINATION ${CMAKE_INSTALL_BINDIR})endif()