LLM inference in C/C++
C++ ggml
ci: Add support for CUDA 13.4 ARM64 builds for Windows (#26650)
shivamkumard-ctrl2e2d99cfd27d
diff --git a/.github/actions/windows-setup-cuda/action.yml b/.github/actions/windows-setup-cuda/action.ymlindex 43c63ce44f..31250eda1b 100644--- a/.github/actions/windows-setup-cuda/action.yml+++ b/.github/actions/windows-setup-cuda/action.yml@@ -4,6 +4,10 @@ inputs:cuda_version:description: "CUDA toolkit version"required: true+ cuda_arch:+ description: "CUDA target architecture"+ required: false+ default: "x64"runs:using: "composite"@@ -127,3 +131,26 @@ runs:echo "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.3\bin" | Out-File -FilePath $env:GITHUB_PATH -Encoding utf8 -Appendecho "CUDA_PATH=C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.3" | Out-File -FilePath $env:GITHUB_ENV -Append -Encoding utf8echo "CUDA_PATH_V13_3=C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.3" | Out-File -FilePath $env:GITHUB_ENV -Append -Encoding utf8++ - name: Install Cuda Toolkit 13.4 for ARM64+ if: ${{ inputs.cuda_version == '13.4' && inputs.cuda_arch == 'arm64' }}+ shell: pwsh+ run: |+ mkdir -p "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.4"+ choco install unzip -y+ curl -O "https://packages.nvidia.com/bin-archive/pool/windows-x86_64/5B515474-7E78-11F1-8656-C51E4F4B317F/cccl-windows-x86_64-13.3.4.1.2-archive.zip"+ curl -O "https://packages.nvidia.com/bin-archive/pool/windows-x86_64/5B515474-7E78-11F1-8656-C51E4F4B317F/cuda_crt-windows-x86_64-13.4.46-archive.zip"+ curl -O "https://packages.nvidia.com/bin-archive/pool/windows-x86_64/5B515474-7E78-11F1-8656-C51E4F4B317F/cuda_nvcc-windows-x86_64-13.4.46-archive.zip"+ curl -O "https://packages.nvidia.com/bin-archive/pool/windows-x86_64/5B515474-7E78-11F1-8656-C51E4F4B317F/libnvvm-windows-x86_64-13.4.46-archive.zip"+ curl -O "https://packages.nvidia.com/bin-archive/pool/windows-arm64/5B515474-7E78-11F1-8656-C51E4F4B317F/cuda_cudart-windows-arm64-13.4.46-archive.zip"+ curl -O "https://packages.nvidia.com/bin-archive/pool/windows-arm64/5B515474-7E78-11F1-8656-C51E4F4B317F/libcublas-windows-arm64-13.7.0.10-archive.zip"+ unzip '*.zip' -d "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.4"+ xcopy "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.4\cccl-windows-x86_64-13.3.4.1.2-archive\*" "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.4" /E /I /H /Y+ xcopy "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.4\cuda_crt-windows-x86_64-13.4.46-archive\*" "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.4" /E /I /H /Y+ xcopy "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.4\cuda_nvcc-windows-x86_64-13.4.46-archive\*" "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.4" /E /I /H /Y+ xcopy "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.4\libnvvm-windows-x86_64-13.4.46-archive\*" "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.4" /E /I /H /Y+ xcopy "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.4\cuda_cudart-windows-arm64-13.4.46-archive\*" "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.4" /E /I /H /Y+ xcopy "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.4\libcublas-windows-arm64-13.7.0.10-archive\*" "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.4" /E /I /H /Y+ echo "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.4\bin" | Out-File -FilePath $env:GITHUB_PATH -Encoding utf8 -Append+ echo "CUDA_PATH=C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.4" | Out-File -FilePath $env:GITHUB_ENV -Append -Encoding utf8+ echo "CUDA_PATH_V13_4=C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.4" | Out-File -FilePath $env:GITHUB_ENV -Append -Encoding utf8diff --git a/.github/workflows/release.yml b/.github/workflows/release.ymlindex 968d2d4b7e..c668930b0c 100644--- a/.github/workflows/release.yml+++ b/.github/workflows/release.yml@@ -848,6 +848,7 @@ jobs:name: llama-bin-win-${{ matrix.backend }}-${{ matrix.arch }}.zipwindows-cuda:+ name: windows-cuda (${{ matrix.cuda }}, ${{ matrix.arch }})needs: [check-release]if: ${{ needs.check-release.outputs.should_release == 'true' }}@@ -858,7 +859,16 @@ jobs:strategy:matrix:- cuda: ['12.4', '13.3']+ include:+ - cuda: '12.4'+ arch: x64+ defines: '-DGGML_CUDA_CUB_3DOT2=ON'+ - cuda: '13.3'+ arch: x64+ defines: ''+ - cuda: '13.4'+ arch: arm64+ defines: '-DCMAKE_TOOLCHAIN_FILE=cmake/arm64-windows-msvc-cuda.cmake'steps:- name: Clone@@ -876,6 +886,7 @@ jobs:uses: ./.github/actions/windows-setup-cudawith:cuda_version: ${{ matrix.cuda }}+ cuda_arch: ${{ matrix.arch }}- name: Install Ninjaid: install_ninja@@ -885,54 +896,62 @@ jobs:- name: ccacheuses: ggml-org/[email protected]with:- key: release-windows-2022-x64-cuda-${{ matrix.cuda }}+ key: release-windows-2022-${{ matrix.arch }}-cuda-${{ matrix.cuda }}- name: Buildid: cmake_buildshell: cmd# TODO: Remove GGML_CUDA_CUB_3DOT2 flag once CCCL 3.2 is bundled within CTK and that CTK version is used in this projectrun: |- call "C:\Program Files\Microsoft Visual Studio\2022\Enterprise\VC\Auxiliary\Build\vcvarsall.bat" x64+ call "C:\Program Files\Microsoft Visual Studio\2022\Enterprise\VC\Auxiliary\Build\vcvarsall.bat" ${{ matrix.arch == 'x64' && 'x64' || 'amd64_arm64' }}cmake -S . -B build -G "Ninja Multi-Config" ^-DGGML_BACKEND_DL=ON ^-DGGML_NATIVE=OFF ^-DGGML_CPU=OFF ^-DGGML_CUDA=ON ^- -DLLAMA_BUILD_BORINGSSL=ON ^- -DGGML_CUDA_CUB_3DOT2=ON+ -DLLAMA_BUILD_BORINGSSL=ON ${{ matrix.defines }}set /A NINJA_JOBS=%NUMBER_OF_PROCESSORS%-1cmake --build build --config Release -j %NINJA_JOBS% --target ggml-cuda- name: ccache-clearuses: ./.github/actions/ccache-clearwith:- key: release-windows-2022-x64-cuda-${{ matrix.cuda }}+ key: release-windows-2022-${{ matrix.arch }}-cuda-${{ matrix.cuda }}- name: Pack artifactsid: pack_artifactsrun: |- 7z a -snl llama-bin-win-cuda-${{ matrix.cuda }}-x64.zip .\build\bin\Release\ggml-cuda.dll+ 7z a -snl llama-bin-win-cuda-${{ matrix.cuda }}-${{ matrix.arch }}.zip .\build\bin\Release\ggml-cuda.dll- name: Upload artifactsuses: actions/upload-artifact@v6with:- path: llama-bin-win-cuda-${{ matrix.cuda }}-x64.zip- name: llama-bin-win-cuda-${{ matrix.cuda }}-x64.zip+ path: llama-bin-win-cuda-${{ matrix.cuda }}-${{ matrix.arch }}.zip+ name: llama-bin-win-cuda-${{ matrix.cuda }}-${{ matrix.arch }}.zip- - name: Copy and pack Cuda runtime+ - name: Copy and pack Cuda runtime (x64)+ if: ${{ matrix.arch == 'x64' }}run: |echo "Cuda install location: ${{ env.CUDA_PATH }}"$dst='.\build\bin\cudart\'robocopy "${{env.CUDA_PATH}}\bin" $dst cudart64_*.dll cublas64_*.dll cublasLt64_*.dllrobocopy "${{env.CUDA_PATH}}\lib" $dst cudart64_*.dll cublas64_*.dll cublasLt64_*.dllrobocopy "${{env.CUDA_PATH}}\bin\x64" $dst cudart64_*.dll cublas64_*.dll cublasLt64_*.dll- 7z a cudart-llama-bin-win-cuda-${{ matrix.cuda }}-x64.zip $dst\*+ 7z a cudart-llama-bin-win-cuda-${{ matrix.cuda }}-${{ matrix.arch }}.zip $dst\*++ - name: Copy and pack Cuda runtime (ARM64)+ if: ${{ matrix.arch == 'arm64' }}+ run: |+ echo "Cuda install location: ${{ env.CUDA_PATH }}"+ $dst='.\build\bin\cudart\'+ robocopy "${{env.CUDA_PATH}}\bin\arm64" $dst cudart64_*.dll cublas64_*.dll cublasLt64_*.dll+ 7z a cudart-llama-bin-win-cuda-${{ matrix.cuda }}-${{ matrix.arch }}.zip $dst\*- name: Upload Cuda runtimeuses: actions/upload-artifact@v6with:- path: cudart-llama-bin-win-cuda-${{ matrix.cuda }}-x64.zip- name: cudart-llama-bin-win-cuda-${{ matrix.cuda }}-x64.zip+ path: cudart-llama-bin-win-cuda-${{ matrix.cuda }}-${{ matrix.arch }}.zip+ name: cudart-llama-bin-win-cuda-${{ matrix.cuda }}-${{ matrix.arch }}.zipwindows-sycl:needs: [check-release]@@ -1679,6 +1698,7 @@ jobs:- [Windows arm64 (OpenCL Adreno)](https://github.com/ggml-org/llama.cpp/releases/download/${{ steps.tag.outputs.name }}/llama-${{ steps.tag.outputs.name }}-bin-win-opencl-adreno-arm64.zip)- [Windows x64 (CUDA 12)](https://github.com/ggml-org/llama.cpp/releases/download/${{ steps.tag.outputs.name }}/llama-${{ steps.tag.outputs.name }}-bin-win-cuda-12.4-x64.zip) - [CUDA 12.4 DLLs](https://github.com/ggml-org/llama.cpp/releases/download/${{ steps.tag.outputs.name }}/cudart-llama-bin-win-cuda-12.4-x64.zip)- [Windows x64 (CUDA 13)](https://github.com/ggml-org/llama.cpp/releases/download/${{ steps.tag.outputs.name }}/llama-${{ steps.tag.outputs.name }}-bin-win-cuda-13.3-x64.zip) - [CUDA 13.3 DLLs](https://github.com/ggml-org/llama.cpp/releases/download/${{ steps.tag.outputs.name }}/cudart-llama-bin-win-cuda-13.3-x64.zip)+ - [Windows arm64 (CUDA 13) (preview)](https://github.com/ggml-org/llama.cpp/releases/download/${{ steps.tag.outputs.name }}/llama-${{ steps.tag.outputs.name }}-bin-win-cuda-13.4-arm64.zip) - [CUDA 13.4 DLLs](https://github.com/ggml-org/llama.cpp/releases/download/${{ steps.tag.outputs.name }}/cudart-llama-bin-win-cuda-13.4-arm64.zip)- [Windows x64 (Vulkan)](https://github.com/ggml-org/llama.cpp/releases/download/${{ steps.tag.outputs.name }}/llama-${{ steps.tag.outputs.name }}-bin-win-vulkan-x64.zip)- [Windows x64 (OpenVINO)](https://github.com/ggml-org/llama.cpp/releases/download/${{ steps.tag.outputs.name }}/llama-${{ steps.tag.outputs.name }}-bin-win-openvino-${{ needs.windows-openvino.outputs.openvino_version }}-x64.zip)- [Windows x64 (SYCL)](https://github.com/ggml-org/llama.cpp/releases/download/${{ steps.tag.outputs.name }}/llama-${{ steps.tag.outputs.name }}-bin-win-sycl-x64.zip)diff --git a/cmake/arm64-windows-msvc-cuda.cmake b/cmake/arm64-windows-msvc-cuda.cmakenew file mode 100644index 0000000000..370f2b3d21--- /dev/null+++ b/cmake/arm64-windows-msvc-cuda.cmake@@ -0,0 +1,26 @@+# Used to cross-compile ggml-cuda for Windows ARM64 on an x64 Windows host.+set( CMAKE_SYSTEM_NAME Windows )+set( CMAKE_SYSTEM_PROCESSOR arm64 )++if ( DEFINED CUDAToolkit_ROOT )+ file( TO_CMAKE_PATH "${CUDAToolkit_ROOT}" CUDA_ROOT )+elseif ( DEFINED ENV{CUDA_PATH} )+ file( TO_CMAKE_PATH "$ENV{CUDA_PATH}" CUDA_ROOT )+else()+ message( FATAL_ERROR "Set CUDAToolkit_ROOT or CUDA_PATH to a Windows CUDA Toolkit with ARM64 target libraries" )+endif()++if ( DEFINED ENV{VCToolsInstallDir} )+ file( TO_CMAKE_PATH "$ENV{VCToolsInstallDir}" MSVC_TOOLS_ROOT )+ set( CMAKE_CUDA_HOST_COMPILER "${MSVC_TOOLS_ROOT}/bin/Hostx64/arm64/cl.exe" CACHE FILEPATH "" )+endif()++set( CMAKE_CUDA_COMPILER "${CUDA_ROOT}/bin/nvcc.exe" CACHE FILEPATH "" )+set( CMAKE_CUDA_FLAGS_INIT "-target-dir=arm64" )++# FindCUDAToolkit selects lib/x64 from the host architecture on Windows.+set( CUDA_CUDART "${CUDA_ROOT}/lib/arm64/cudart.lib" CACHE FILEPATH "" )+set( CUDA_cudart_LIBRARY "${CUDA_ROOT}/lib/arm64/cudart.lib" CACHE FILEPATH "" )+set( CUDA_cublas_LIBRARY "${CUDA_ROOT}/lib/arm64/cublas.lib" CACHE FILEPATH "" )+set( CUDA_cublasLt_LIBRARY "${CUDA_ROOT}/lib/arm64/cublasLt.lib" CACHE FILEPATH "" )+set( CUDA_cuda_driver_LIBRARY "${CUDA_ROOT}/lib/arm64/cuda.lib" CACHE FILEPATH "" )