LLM inference in C/C++
C++ ggml
ci: more optimizations (#26983)
Eve058df671b214
diff --git a/.github/actions/linux-setup-vulkan/action.yml b/.github/actions/linux-setup-vulkan/action.ymldeleted file mode 100644index 4d29837feb..0000000000--- a/.github/actions/linux-setup-vulkan/action.yml+++ /dev/null@@ -1,20 +0,0 @@-name: "Linux - Setup Vulkan SDK"-description: "Setup Vulkan SDK for Linux"-inputs:- path:- description: "Installation path"- required: true- version:- description: "Vulkan SDK version"- required: true--runs:- using: "composite"- steps:- - name: Setup Vulkan SDK- id: setup- uses: ./.github/actions/unarchive-tar- with:- url: https://sdk.lunarg.com/sdk/download/${{ inputs.version }}/linux/vulkan_sdk.tar.xz- path: ${{ inputs.path }}- strip: 1diff --git a/.github/workflows/build-cache.yml b/.github/workflows/build-cache.ymlindex 2a10317285..604f842410 100644--- a/.github/workflows/build-cache.yml+++ b/.github/workflows/build-cache.yml@@ -10,33 +10,6 @@ concurrency:cancel-in-progress: truejobs:- ubuntu-24-vulkan-cache:- runs-on: ubuntu-24.04-- steps:- - name: Clone- id: checkout- uses: actions/checkout@v6-- - name: Get latest Vulkan SDK version- id: vulkan_sdk_version- run: |- echo "VULKAN_SDK_VERSION=$(curl https://vulkan.lunarg.com/sdk/latest/linux.txt)" >> "$GITHUB_ENV"-- - name: Setup Cache- uses: actions/cache@v5- id: cache-sdk- with:- path: ./vulkan_sdk- key: cache-gha-vulkan-sdk-${{ env.VULKAN_SDK_VERSION }}-${{ runner.os }}-- - name: Setup Vulkan SDK- if: steps.cache-sdk.outputs.cache-hit != 'true'- uses: ./.github/actions/linux-setup-vulkan- with:- path: ./vulkan_sdk- version: ${{ env.VULKAN_SDK_VERSION }}-#ubuntu-24-spacemit-cache:# runs-on: ubuntu-24.04diff --git a/.github/workflows/build-cpu.yml b/.github/workflows/build-cpu.ymlindex df70f4d10b..2016a57f87 100644--- a/.github/workflows/build-cpu.yml+++ b/.github/workflows/build-cpu.yml@@ -21,6 +21,7 @@ on:paths: ['.github/workflows/build-cpu.yml','.github/workflows/build-cmake-pkg.yml',+ 'ggml/src/ggml-rpc/**','**/CMakeLists.txt','**/.cmake','**/*.h',@@ -123,7 +124,6 @@ jobs:env:OPENBLAS_VERSION: 0.3.23SDE_VERSION: 9.33.0-2024-01-07- VULKAN_VERSION: 1.4.357.0strategy:matrix:@@ -134,9 +134,6 @@ jobs:- build: 'x64-openblas'arch: 'x64'defines: '-G "Ninja Multi-Config" -D CMAKE_TOOLCHAIN_FILE=cmake/x64-windows-llvm.cmake -DGGML_NATIVE=OFF -DLLAMA_BUILD_SERVER=ON -DGGML_RPC=ON -DGGML_BACKEND_DL=ON -DGGML_CPU_ALL_VARIANTS=ON -DGGML_OPENMP=OFF -DGGML_BLAS=ON -DGGML_BLAS_VENDOR=OpenBLAS -DBLAS_INCLUDE_DIRS="$env:RUNNER_TEMP/openblas/include" -DBLAS_LIBRARIES="$env:RUNNER_TEMP/openblas/lib/openblas.lib"'- - build: 'x64-vulkan'- arch: 'x64'- defines: '-G "Ninja Multi-Config" -D CMAKE_TOOLCHAIN_FILE=cmake/x64-windows-llvm.cmake -DCMAKE_BUILD_TYPE=Release -DGGML_NATIVE=OFF -DLLAMA_BUILD_SERVER=ON -DGGML_RPC=ON -DGGML_BACKEND_DL=ON -DGGML_CPU_ALL_VARIANTS=ON -DGGML_VULKAN=ON'- build: 'arm64'arch: 'arm64'defines: '-G "Ninja Multi-Config" -D CMAKE_TOOLCHAIN_FILE=cmake/arm64-windows-llvm.cmake -DGGML_NATIVE=OFF -DLLAMA_BUILD_SERVER=ON'@@ -167,15 +164,6 @@ jobs:$lib = $(join-path $msvc 'bin\Hostx64\x64\lib.exe')& $lib /machine:x64 "/def:${env:RUNNER_TEMP}/openblas/lib/libopenblas.def" "/out:${env:RUNNER_TEMP}/openblas/lib/openblas.lib" /name:openblas.dll- - name: Install Vulkan SDK- id: get_vulkan- if: ${{ matrix.build == 'x64-vulkan' }}- run: |- curl.exe -o $env:RUNNER_TEMP/VulkanSDK-Installer.exe -L "https://sdk.lunarg.com/sdk/download/${env:VULKAN_VERSION}/windows/vulkansdk-windows-X64-${env:VULKAN_VERSION}.exe"- & "$env:RUNNER_TEMP\VulkanSDK-Installer.exe" --accept-licenses --default-answer --confirm-command install- Add-Content $env:GITHUB_ENV "VULKAN_SDK=C:\VulkanSDK\${env:VULKAN_VERSION}"- Add-Content $env:GITHUB_PATH "C:\VulkanSDK\${env:VULKAN_VERSION}\bin"-- name: Install Ninjaid: install_ninjarun: |diff --git a/.github/workflows/build-rpc.yml b/.github/workflows/build-rpc.ymldeleted file mode 100644index d04dc375b5..0000000000--- a/.github/workflows/build-rpc.yml+++ /dev/null@@ -1,66 +0,0 @@-name: CI (rpc)--on:- workflow_dispatch: # allows manual triggering- push:- branches:- - master- paths: [- '.github/workflows/build-rpc.yml',- '**/CMakeLists.txt',- '**/.cmake',- '**/*.h',- '**/*.hpp',- '**/*.c',- '**/*.cpp'- ]-- pull_request:- types: [opened, synchronize, reopened]- paths: [- '.github/workflows/build-rpc.yml',- 'ggml/src/ggml-rpc/**'- ]--concurrency:- group: ${{ github.workflow }}-${{ github.head_ref && github.ref || github.run_id }}- cancel-in-progress: true--env:- GGML_NLOOP: 3- GGML_N_THREADS: 1- LLAMA_ARG_LOG_COLORS: 1- LLAMA_ARG_LOG_PREFIX: 1- LLAMA_ARG_LOG_TIMESTAMPS: 1--jobs:- ubuntu-24-rpc:- runs-on: ${{ 'ubuntu-24.04-arm' || 'ubuntu-24.04' }}-- continue-on-error: true-- steps:- - name: Clone- id: checkout- uses: actions/checkout@v6-- - name: Dependencies- id: depends- run: |- sudo apt-get update- sudo apt-get install build-essential libssl-dev ninja-build-- - name: Build- id: cmake_build- run: |- cmake -B build \- -G "Ninja" \- -DCMAKE_BUILD_TYPE=Release \- -DGGML_RPC=ON- time cmake --build build --config Release -j $(nproc)-- - name: Test- id: cmake_test- run: |- cd build- ctest -L main --verbosediff --git a/.github/workflows/build-vulkan.yml b/.github/workflows/build-vulkan.ymlindex 01113803ff..15ff4b0af3 100644--- a/.github/workflows/build-vulkan.yml+++ b/.github/workflows/build-vulkan.yml@@ -93,19 +93,13 @@ jobs:run: |echo "VULKAN_SDK_VERSION=$(curl https://vulkan.lunarg.com/sdk/latest/linux.txt)" >> "$GITHUB_ENV"- - name: Use Vulkan SDK Cache- uses: actions/cache@v5- id: cache-sdk- with:- path: ./vulkan_sdk- key: cache-gha-vulkan-sdk-${{ env.VULKAN_SDK_VERSION }}-${{ runner.os }}-- name: Setup Vulkan SDK- if: steps.cache-sdk.outputs.cache-hit != 'true'- uses: ./.github/actions/linux-setup-vulkan+ id: setup+ uses: ./.github/actions/unarchive-tarwith:+ url: https://sdk.lunarg.com/sdk/download/${{ env.VULKAN_SDK_VERSION }}/linux/vulkan_sdk.tar.xzpath: ./vulkan_sdk- version: ${{ env.VULKAN_SDK_VERSION }}+ strip: 1- name: ccacheuses: ggml-org/[email protected]@@ -133,3 +127,56 @@ jobs:# This is using llvmpipe and runs slower than other backends# test-backend-ops is too slow on llvmpipe, skip itctest -L main -E test-backend-ops --verbose --timeout 900++ windows:+ runs-on: windows-2025++ env:+ VULKAN_VERSION: 1.4.357.0++ steps:+ - name: Clone+ id: checkout+ uses: actions/checkout@v6++ - name: ccache+ uses: ggml-org/[email protected]+ with:+ key: cpu-windows-2025-x64-vulkan+ variant: ccache+ evict-old-files: 1d+ save: ${{ github.event_name == 'push' && github.ref == 'refs/heads/master' }}++ - name: Install Vulkan SDK+ id: get_vulkan+ run: |+ curl.exe -o $env:RUNNER_TEMP/VulkanSDK-Installer.exe -L "https://sdk.lunarg.com/sdk/download/${env:VULKAN_VERSION}/windows/vulkansdk-windows-X64-${env:VULKAN_VERSION}.exe"+ & "$env:RUNNER_TEMP\VulkanSDK-Installer.exe" --accept-licenses --default-answer --confirm-command install+ Add-Content $env:GITHUB_ENV "VULKAN_SDK=C:\VulkanSDK\${env:VULKAN_VERSION}"+ Add-Content $env:GITHUB_PATH "C:\VulkanSDK\${env:VULKAN_VERSION}\bin"++ - name: Install Ninja+ id: install_ninja+ run: |+ choco install ninja++ - name: Build+ id: cmake_build+ run: |+ cmake -S . -B build -G "Ninja Multi-Config" `+ -D CMAKE_TOOLCHAIN_FILE=cmake/x64-windows-llvm.cmake `+ -DCMAKE_BUILD_TYPE=Release `+ -DGGML_NATIVE=OFF `+ -DLLAMA_BUILD_SERVER=ON `+ -DGGML_RPC=ON `+ -DGGML_BACKEND_DL=ON `+ -DGGML_CPU_ALL_VARIANTS=ON `+ -DGGML_VULKAN=ON `+ -DLLAMA_BUILD_BORINGSSL=ON+ cmake --build build --config Release -j ${env:NUMBER_OF_PROCESSORS}++ - name: Test+ id: cmake_test+ run: |+ cd build+ ctest -L main -C Release --verbose --timeout 900