LLM inference in C/C++
C++ ggml
CI: Use LLVM's OpenMP over MSVC_DEBUG_non_redist on Windows (#26678)
Oliver Simonsbf0040e15fd5
diff --git a/.github/workflows/build-cpu.yml b/.github/workflows/build-cpu.ymlindex 2016a57f87..a63ffe94b4 100644--- a/.github/workflows/build-cpu.yml+++ b/.github/workflows/build-cpu.yml@@ -119,6 +119,7 @@ jobs:./bin/llama-completion -m stories260K.gguf -p "One day, Lily met a Shoggoth" -n 500 -c 256windows:+ name: windows / ${{ matrix.build }}runs-on: windows-2025env:@@ -130,13 +131,13 @@ jobs:include:- build: 'x64-cpu-static'arch: 'x64'- defines: '-G "Ninja Multi-Config" -D CMAKE_TOOLCHAIN_FILE=cmake/x64-windows-llvm.cmake -DGGML_NATIVE=OFF -DLLAMA_BUILD_SERVER=ON -DGGML_RPC=ON -DBUILD_SHARED_LIBS=OFF'+ defines: '-G "Ninja Multi-Config" -D CMAKE_TOOLCHAIN_FILE=cmake/x64-windows-llvm.cmake -DGGML_NATIVE=OFF -DGGML_OPENMP_FETCH=ON -DLLAMA_BUILD_SERVER=ON -DGGML_RPC=ON -DBUILD_SHARED_LIBS=OFF'- build: 'x64-openblas'arch: 'x64'defines: '-G "Ninja Multi-Config" -D CMAKE_TOOLCHAIN_FILE=cmake/x64-windows-llvm.cmake -DGGML_NATIVE=OFF -DLLAMA_BUILD_SERVER=ON -DGGML_RPC=ON -DGGML_BACKEND_DL=ON -DGGML_CPU_ALL_VARIANTS=ON -DGGML_OPENMP=OFF -DGGML_BLAS=ON -DGGML_BLAS_VENDOR=OpenBLAS -DBLAS_INCLUDE_DIRS="$env:RUNNER_TEMP/openblas/include" -DBLAS_LIBRARIES="$env:RUNNER_TEMP/openblas/lib/openblas.lib"'- build: 'arm64'arch: 'arm64'- defines: '-G "Ninja Multi-Config" -D CMAKE_TOOLCHAIN_FILE=cmake/arm64-windows-llvm.cmake -DGGML_NATIVE=OFF -DLLAMA_BUILD_SERVER=ON'+ defines: '-G "Ninja Multi-Config" -D CMAKE_TOOLCHAIN_FILE=cmake/arm64-windows-llvm.cmake -DGGML_NATIVE=OFF -DGGML_OPENMP_FETCH=ON -DLLAMA_BUILD_SERVER=ON'steps:- name: Clonediff --git a/.github/workflows/release.yml b/.github/workflows/release.ymlindex 20c9690869..61b2f5485d 100644--- a/.github/workflows/release.yml+++ b/.github/workflows/release.yml@@ -681,6 +681,7 @@ jobs:name: llama-bin-win-openvino-${{ env.OPENVINO_VERSION_MAJOR }}-x64.zipwindows-cpu:+ name: windows-cpu / ${{ matrix.arch }}needs: [check-release]if: ${{ needs.check-release.outputs.should_release == 'true' }}@@ -728,6 +729,7 @@ jobs:-DGGML_BACKEND_DL=ON ^-DGGML_CPU_ALL_VARIANTS=${{ matrix.arch == 'x64' && 'ON' || 'OFF' }} ^-DGGML_OPENMP=ON ^+ -DGGML_OPENMP_FETCH=ON ^${{ env.CMAKE_ARGS }}cmake --build build --config Release@@ -739,7 +741,6 @@ jobs:- name: Pack artifactsid: pack_artifactsrun: |- Copy-Item "C:\Program Files\Microsoft Visual Studio\18\Enterprise\VC\Redist\MSVC\14.51.36231\debug_nonredist\${{ matrix.arch }}\Microsoft.VC145.OpenMP.LLVM\libomp140.${{ matrix.arch == 'x64' && 'x86_64' || 'aarch64' }}.dll" .\build\bin\Release\7z a -snl llama-bin-win-cpu-${{ matrix.arch }}.zip .\build\bin\Release\*- name: Upload artifactsdiff --git a/cmake/arm64-windows-llvm.cmake b/cmake/arm64-windows-llvm.cmakeindex 8023796800..cdba4e7494 100644--- a/cmake/arm64-windows-llvm.cmake+++ b/cmake/arm64-windows-llvm.cmake@@ -8,6 +8,7 @@ set( CMAKE_CXX_COMPILER clang++ )set( CMAKE_C_COMPILER_TARGET ${target} )set( CMAKE_CXX_COMPILER_TARGET ${target} )+set( CMAKE_ASM_COMPILER_TARGET ${target} )set( arch_c_flags "-march=armv8.7-a -fvectorize -ffp-model=fast -fno-finite-math-only" )set( warn_c_flags "-Wno-format -Wno-unused-variable -Wno-unused-function -Wno-gnu-zero-variadic-macro-arguments" )diff --git a/docs/build.md b/docs/build.mdindex ca086a0be1..45fe7f17a2 100644--- a/docs/build.md+++ b/docs/build.md@@ -72,9 +72,10 @@ cmake --build build --config Release- Please remember to always use a Developer Command Prompt / PowerShell for VS2022 for git, build, test- For Windows on ARM (arm64, WoA) build with:```bash- cmake --preset arm64-windows-llvm-release -D GGML_OPENMP=OFF+ cmake --preset arm64-windows-llvm-release -D GGML_OPENMP_FETCH=ONcmake --build build-arm64-windows-llvm-release```+ `GGML_OPENMP_FETCH` downloads the official LLVM OpenMP runtime and requires Clang, 7-Zip and network access during configuration. CMake selects the runtime from the target architecture, so this also works when cross-compiling for WoA from x64. The extracted header, import library, DLL and OpenMP license are placed under `build/_deps`. The build copies `libomp.dll` and `LICENSE-LLVM-OpenMP` to the runtime output directory and installs them together. Omit the option to use CMake's normal OpenMP detection, or pass `-D GGML_OPENMP=OFF` to disable OpenMP.For building with ninja generator and clang compiler as default:-set path:set LIB=C:\Program Files (x86)\Windows Kits\10\Lib\10.0.22621.0\um\x64;C:\Program Files\Microsoft Visual Studio\2022\Community\VC\Tools\MSVC\14.41.34120\lib\x64\uwp;C:\Program Files (x86)\Windows Kits\10\Lib\10.0.22621.0\ucrt\x64```bashdiff --git a/ggml/CMakeLists.txt b/ggml/CMakeLists.txtindex b7110fa12b..9d807d5c83 100644--- a/ggml/CMakeLists.txt+++ b/ggml/CMakeLists.txt@@ -243,6 +243,7 @@ set (GGML_METAL_MACOSX_VERSION_MIN "" CACHE STRING"ggml: metal minimum macOS version")set (GGML_METAL_STD "" CACHE STRING "ggml: metal standard version (-std flag)")option(GGML_OPENMP "ggml: use OpenMP" ON)+option(GGML_OPENMP_FETCH "ggml: fetch LLVM OpenMP" OFF)option(GGML_RPC "ggml: use RPC" OFF)option(GGML_SYCL "ggml: use SYCL" OFF)option(GGML_SYCL_F16 "ggml: use 16 bit floats for sycl calculations" OFF)diff --git a/ggml/src/CMakeLists.txt b/ggml/src/CMakeLists.txtindex 82e9480c2f..96535b49fa 100644--- a/ggml/src/CMakeLists.txt+++ b/ggml/src/CMakeLists.txt@@ -222,9 +222,123 @@ if (GGML_SCHED_NO_REALLOC)target_compile_definitions(ggml-base PUBLIC GGML_SCHED_NO_REALLOC)endif()-if (GGML_OPENMP)+if (GGML_OPENMP_FETCH)+ if (NOT GGML_OPENMP)+ message(FATAL_ERROR "GGML_OPENMP_FETCH requires GGML_OPENMP")+ elseif (NOT WIN32 OR NOT (CMAKE_C_COMPILER_ID MATCHES "Clang"))+ message(FATAL_ERROR "GGML_OPENMP_FETCH currently requires Clang on Windows")+ endif()++ set(GGML_OPENMP_LLVM_VERSION "20.1.8")+ string(REGEX MATCH "^[0-9]+" GGML_OPENMP_LLVM_VERSION_MAJOR "${GGML_OPENMP_LLVM_VERSION}")+ string(REGEX MATCH "^[0-9]+" GGML_OPENMP_COMPILER_VERSION_MAJOR "${CMAKE_C_COMPILER_VERSION}")+ if (NOT GGML_OPENMP_COMPILER_VERSION_MAJOR STREQUAL GGML_OPENMP_LLVM_VERSION_MAJOR)+ message(FATAL_ERROR "LLVM OpenMP ${GGML_OPENMP_LLVM_VERSION} requires Clang ${GGML_OPENMP_LLVM_VERSION_MAJOR}.x")+ endif()++ string(TOLOWER "${CMAKE_SYSTEM_PROCESSOR}" GGML_OPENMP_SYSTEM_PROCESSOR)+ if (GGML_OPENMP_SYSTEM_PROCESSOR MATCHES "^(amd64|x86_64)$")+ set(GGML_OPENMP_ARCH "x64")+ set(GGML_OPENMP_INSTALLER_SUFFIX "win64")+ set(GGML_OPENMP_INSTALLER_SHA256 "3197846a2b19063687dd56e93e34cd941e3548d907f23a6131571321bdf9fe7b")+ elseif (GGML_OPENMP_SYSTEM_PROCESSOR MATCHES "^(aarch64|arm64)$")+ set(GGML_OPENMP_ARCH "arm64")+ set(GGML_OPENMP_INSTALLER_SUFFIX "woa64")+ set(GGML_OPENMP_INSTALLER_SHA256 "7c4ac97eb2ae6b960ca5f9caf3ff6124c8d2a18cc07a7840a4d2ea15537bad8e")+ else()+ message(FATAL_ERROR "GGML_OPENMP_FETCH does not support ${CMAKE_SYSTEM_PROCESSOR}")+ endif()++ set(GGML_OPENMP_CACHE_DIR "${CMAKE_BINARY_DIR}/_deps")+ set(GGML_OPENMP_ROOT "${GGML_OPENMP_CACHE_DIR}/llvm-openmp-${GGML_OPENMP_LLVM_VERSION}-${GGML_OPENMP_ARCH}")+ set(GGML_OPENMP_LIBRARY "${GGML_OPENMP_ROOT}/lib/libomp.lib")+ set(GGML_OPENMP_RUNTIME "${GGML_OPENMP_ROOT}/bin/libomp.dll")+ set(GGML_OPENMP_HEADER "${GGML_OPENMP_ROOT}/include/omp.h")+ set(GGML_OPENMP_LICENSE "${GGML_OPENMP_ROOT}/LICENSE.TXT")+ set(GGML_OPENMP_LICENSE_SHA256 "fdad1758a9e1f9d5a81e18879b3406772115edc92c24bfa36b70c654f325e8e4")++ if (NOT EXISTS "${GGML_OPENMP_LIBRARY}" OR NOT EXISTS "${GGML_OPENMP_RUNTIME}" OR NOT EXISTS "${GGML_OPENMP_HEADER}")+ find_program(GGML_OPENMP_7Z NAMES 7z 7zz 7za)+ if (NOT GGML_OPENMP_7Z)+ message(FATAL_ERROR "GGML_OPENMP_FETCH requires 7-Zip to extract the LLVM installer")+ endif()++ set(GGML_OPENMP_INSTALLER "${GGML_OPENMP_ROOT}/LLVM-${GGML_OPENMP_LLVM_VERSION}-${GGML_OPENMP_INSTALLER_SUFFIX}.exe")+ set(GGML_OPENMP_EXTRACT_DIR "${GGML_OPENMP_ROOT}/extract")+ set(GGML_OPENMP_INSTALLER_URL "https://github.com/llvm/llvm-project/releases/download/llvmorg-${GGML_OPENMP_LLVM_VERSION}/LLVM-${GGML_OPENMP_LLVM_VERSION}-${GGML_OPENMP_INSTALLER_SUFFIX}.exe")++ file(MAKE_DIRECTORY "${GGML_OPENMP_EXTRACT_DIR}")+ file(DOWNLOAD "${GGML_OPENMP_INSTALLER_URL}" "${GGML_OPENMP_INSTALLER}"+ EXPECTED_HASH "SHA256=${GGML_OPENMP_INSTALLER_SHA256}"+ SHOW_PROGRESS+ STATUS GGML_OPENMP_DOWNLOAD_STATUS)+ list(GET GGML_OPENMP_DOWNLOAD_STATUS 0 GGML_OPENMP_DOWNLOAD_RESULT)+ if (NOT GGML_OPENMP_DOWNLOAD_RESULT EQUAL 0)+ list(GET GGML_OPENMP_DOWNLOAD_STATUS 1 GGML_OPENMP_DOWNLOAD_ERROR)+ message(FATAL_ERROR "Failed to download LLVM OpenMP: ${GGML_OPENMP_DOWNLOAD_ERROR}")+ endif()++ execute_process(+ COMMAND "${GGML_OPENMP_7Z}" e -y "-o${GGML_OPENMP_EXTRACT_DIR}" "${GGML_OPENMP_INSTALLER}" -r libomp.lib libomp.dll omp.h+ RESULT_VARIABLE GGML_OPENMP_EXTRACT_RESULT+ OUTPUT_QUIET)+ if (NOT GGML_OPENMP_EXTRACT_RESULT EQUAL 0 OR+ NOT EXISTS "${GGML_OPENMP_EXTRACT_DIR}/libomp.lib" OR+ NOT EXISTS "${GGML_OPENMP_EXTRACT_DIR}/libomp.dll" OR+ NOT EXISTS "${GGML_OPENMP_EXTRACT_DIR}/omp.h")+ message(FATAL_ERROR "Failed to extract libomp from ${GGML_OPENMP_INSTALLER}")+ endif()++ file(MAKE_DIRECTORY "${GGML_OPENMP_ROOT}/lib" "${GGML_OPENMP_ROOT}/bin" "${GGML_OPENMP_ROOT}/include")+ file(COPY "${GGML_OPENMP_EXTRACT_DIR}/libomp.lib" DESTINATION "${GGML_OPENMP_ROOT}/lib")+ file(COPY "${GGML_OPENMP_EXTRACT_DIR}/libomp.dll" DESTINATION "${GGML_OPENMP_ROOT}/bin")+ file(COPY "${GGML_OPENMP_EXTRACT_DIR}/omp.h" DESTINATION "${GGML_OPENMP_ROOT}/include")+ file(REMOVE_RECURSE "${GGML_OPENMP_INSTALLER}" "${GGML_OPENMP_EXTRACT_DIR}")+ endif()++ # The NSIS installer embeds LLVM's general license in its UI but does not install it as a file; use OpenMP's license to include its additional notices.+ if (EXISTS "${GGML_OPENMP_LICENSE}")+ file(SHA256 "${GGML_OPENMP_LICENSE}" GGML_OPENMP_LICENSE_ACTUAL_SHA256)+ endif()+ if (NOT GGML_OPENMP_LICENSE_ACTUAL_SHA256 STREQUAL GGML_OPENMP_LICENSE_SHA256)+ file(DOWNLOAD "https://raw.githubusercontent.com/llvm/llvm-project/llvmorg-${GGML_OPENMP_LLVM_VERSION}/openmp/LICENSE.TXT" "${GGML_OPENMP_LICENSE}"+ EXPECTED_HASH "SHA256=${GGML_OPENMP_LICENSE_SHA256}")+ endif()++ if (COMMAND license_add_file)+ license_add_file("LLVM OpenMP" "${GGML_OPENMP_LICENSE}")+ endif()++ add_library(ggml-openmp-c INTERFACE)+ target_compile_options(ggml-openmp-c INTERFACE "$<$<COMPILE_LANGUAGE:C>:-fopenmp=libomp>")+ target_include_directories(ggml-openmp-c SYSTEM INTERFACE "${GGML_OPENMP_ROOT}/include")+ target_link_libraries(ggml-openmp-c INTERFACE "${GGML_OPENMP_LIBRARY}")++ add_library(ggml-openmp-cxx INTERFACE)+ target_compile_options(ggml-openmp-cxx INTERFACE "$<$<COMPILE_LANGUAGE:CXX>:-fopenmp=libomp>")+ target_include_directories(ggml-openmp-cxx SYSTEM INTERFACE "${GGML_OPENMP_ROOT}/include")+ target_link_libraries(ggml-openmp-cxx INTERFACE "${GGML_OPENMP_LIBRARY}")++ set(GGML_OPENMP_RUNTIME_OUTPUT_DIR "${CMAKE_RUNTIME_OUTPUT_DIRECTORY}")+ if (CMAKE_CONFIGURATION_TYPES)+ string(APPEND GGML_OPENMP_RUNTIME_OUTPUT_DIR "/$<CONFIG>")+ endif()+ add_custom_target(ggml-openmp-runtime ALL+ COMMAND ${CMAKE_COMMAND} -E make_directory "${GGML_OPENMP_RUNTIME_OUTPUT_DIR}"+ COMMAND ${CMAKE_COMMAND} -E copy_if_different "${GGML_OPENMP_RUNTIME}" "${GGML_OPENMP_RUNTIME_OUTPUT_DIR}/libomp.dll"+ COMMAND ${CMAKE_COMMAND} -E copy_if_different "${GGML_OPENMP_LICENSE}" "${GGML_OPENMP_RUNTIME_OUTPUT_DIR}/LICENSE-LLVM-OpenMP")+ add_dependencies(ggml-base ggml-openmp-runtime)+ install(FILES "${GGML_OPENMP_RUNTIME}" DESTINATION ${CMAKE_INSTALL_BINDIR})+ install(FILES "${GGML_OPENMP_LICENSE}" DESTINATION ${CMAKE_INSTALL_BINDIR} RENAME LICENSE-LLVM-OpenMP)++ set(GGML_OPENMP_TARGET_C ggml-openmp-c)+ set(GGML_OPENMP_TARGET_CXX ggml-openmp-cxx)+ set(GGML_OPENMP_ENABLED "ON" CACHE INTERNAL "")+elseif (GGML_OPENMP)find_package(OpenMP)if (OpenMP_FOUND)+ set(GGML_OPENMP_TARGET_C OpenMP::OpenMP_C)+ set(GGML_OPENMP_TARGET_CXX OpenMP::OpenMP_CXX)set(GGML_OPENMP_ENABLED "ON" CACHE INTERNAL "")else()set(GGML_OPENMP_ENABLED "OFF" CACHE INTERNAL "")@@ -236,7 +350,7 @@ endif()if (GGML_OPENMP_ENABLED)target_compile_definitions(ggml-base PRIVATE GGML_USE_OPENMP)- target_link_libraries(ggml-base PRIVATE OpenMP::OpenMP_C OpenMP::OpenMP_CXX)+ target_link_libraries(ggml-base PRIVATE ${GGML_OPENMP_TARGET_C} ${GGML_OPENMP_TARGET_CXX})endif()add_library(ggmldiff --git a/ggml/src/ggml-cpu/CMakeLists.txt b/ggml/src/ggml-cpu/CMakeLists.txtindex 836bae4d05..a6cc49586b 100644--- a/ggml/src/ggml-cpu/CMakeLists.txt+++ b/ggml/src/ggml-cpu/CMakeLists.txt@@ -74,7 +74,7 @@ function(ggml_add_cpu_backend_variant_impl tag_name)if (GGML_OPENMP_ENABLED)target_compile_definitions(${GGML_CPU_NAME} PRIVATE GGML_USE_OPENMP)- target_link_libraries(${GGML_CPU_NAME} PRIVATE OpenMP::OpenMP_C OpenMP::OpenMP_CXX)+ target_link_libraries(${GGML_CPU_NAME} PRIVATE ${GGML_OPENMP_TARGET_C} ${GGML_OPENMP_TARGET_CXX})endif()if (GGML_LLAMAFILE)diff --git a/ggml/src/ggml-zendnn/CMakeLists.txt b/ggml/src/ggml-zendnn/CMakeLists.txtindex 87d721f6d7..6e393d6b66 100644--- a/ggml/src/ggml-zendnn/CMakeLists.txt+++ b/ggml/src/ggml-zendnn/CMakeLists.txt@@ -86,6 +86,6 @@ endif()target_link_libraries(ggml-zendnn PRIVATE m pthread)-if (GGML_OPENMP)- target_link_libraries(ggml-zendnn PRIVATE OpenMP::OpenMP_CXX)+if (GGML_OPENMP_ENABLED)+ target_link_libraries(ggml-zendnn PRIVATE ${GGML_OPENMP_TARGET_CXX})endif()