Skip to content

LLM inference in C/C++

C++ ggml

ci: more optimizations (#26983)

Eve058df671b214
diff --git a/.github/actions/linux-setup-vulkan/action.yml b/.github/actions/linux-setup-vulkan/action.yml
deleted file mode 100644
index 4d29837feb..0000000000
--- a/.github/actions/linux-setup-vulkan/action.yml
+++ /dev/null
@@ -1,20 +0,0 @@
-name: "Linux - Setup Vulkan SDK"
-description: "Setup Vulkan SDK for Linux"
-inputs:
- path:
- description: "Installation path"
- required: true
- version:
- description: "Vulkan SDK version"
- required: true
-
-runs:
- using: "composite"
- steps:
- - name: Setup Vulkan SDK
- id: setup
- uses: ./.github/actions/unarchive-tar
- with:
- url: https://sdk.lunarg.com/sdk/download/${{ inputs.version }}/linux/vulkan_sdk.tar.xz
- path: ${{ inputs.path }}
- strip: 1
diff --git a/.github/workflows/build-cache.yml b/.github/workflows/build-cache.yml
index 2a10317285..604f842410 100644
--- a/.github/workflows/build-cache.yml
+++ b/.github/workflows/build-cache.yml
@@ -10,33 +10,6 @@ concurrency:
cancel-in-progress: true
jobs:
- ubuntu-24-vulkan-cache:
- runs-on: ubuntu-24.04
-
- steps:
- - name: Clone
- id: checkout
- uses: actions/checkout@v6
-
- - name: Get latest Vulkan SDK version
- id: vulkan_sdk_version
- run: |
- echo "VULKAN_SDK_VERSION=$(curl https://vulkan.lunarg.com/sdk/latest/linux.txt)" >> "$GITHUB_ENV"
-
- - name: Setup Cache
- uses: actions/cache@v5
- id: cache-sdk
- with:
- path: ./vulkan_sdk
- key: cache-gha-vulkan-sdk-${{ env.VULKAN_SDK_VERSION }}-${{ runner.os }}
-
- - name: Setup Vulkan SDK
- if: steps.cache-sdk.outputs.cache-hit != 'true'
- uses: ./.github/actions/linux-setup-vulkan
- with:
- path: ./vulkan_sdk
- version: ${{ env.VULKAN_SDK_VERSION }}
-
#ubuntu-24-spacemit-cache:
# runs-on: ubuntu-24.04
diff --git a/.github/workflows/build-cpu.yml b/.github/workflows/build-cpu.yml
index df70f4d10b..2016a57f87 100644
--- a/.github/workflows/build-cpu.yml
+++ b/.github/workflows/build-cpu.yml
@@ -21,6 +21,7 @@ on:
paths: [
'.github/workflows/build-cpu.yml',
'.github/workflows/build-cmake-pkg.yml',
+ 'ggml/src/ggml-rpc/**',
'**/CMakeLists.txt',
'**/.cmake',
'**/*.h',
@@ -123,7 +124,6 @@ jobs:
env:
OPENBLAS_VERSION: 0.3.23
SDE_VERSION: 9.33.0-2024-01-07
- VULKAN_VERSION: 1.4.357.0
strategy:
matrix:
@@ -134,9 +134,6 @@ jobs:
- build: 'x64-openblas'
arch: 'x64'
defines: '-G "Ninja Multi-Config" -D CMAKE_TOOLCHAIN_FILE=cmake/x64-windows-llvm.cmake -DGGML_NATIVE=OFF -DLLAMA_BUILD_SERVER=ON -DGGML_RPC=ON -DGGML_BACKEND_DL=ON -DGGML_CPU_ALL_VARIANTS=ON -DGGML_OPENMP=OFF -DGGML_BLAS=ON -DGGML_BLAS_VENDOR=OpenBLAS -DBLAS_INCLUDE_DIRS="$env:RUNNER_TEMP/openblas/include" -DBLAS_LIBRARIES="$env:RUNNER_TEMP/openblas/lib/openblas.lib"'
- - build: 'x64-vulkan'
- arch: 'x64'
- defines: '-G "Ninja Multi-Config" -D CMAKE_TOOLCHAIN_FILE=cmake/x64-windows-llvm.cmake -DCMAKE_BUILD_TYPE=Release -DGGML_NATIVE=OFF -DLLAMA_BUILD_SERVER=ON -DGGML_RPC=ON -DGGML_BACKEND_DL=ON -DGGML_CPU_ALL_VARIANTS=ON -DGGML_VULKAN=ON'
- build: 'arm64'
arch: 'arm64'
defines: '-G "Ninja Multi-Config" -D CMAKE_TOOLCHAIN_FILE=cmake/arm64-windows-llvm.cmake -DGGML_NATIVE=OFF -DLLAMA_BUILD_SERVER=ON'
@@ -167,15 +164,6 @@ jobs:
$lib = $(join-path $msvc 'bin\Hostx64\x64\lib.exe')
& $lib /machine:x64 "/def:${env:RUNNER_TEMP}/openblas/lib/libopenblas.def" "/out:${env:RUNNER_TEMP}/openblas/lib/openblas.lib" /name:openblas.dll
- - name: Install Vulkan SDK
- id: get_vulkan
- if: ${{ matrix.build == 'x64-vulkan' }}
- run: |
- curl.exe -o $env:RUNNER_TEMP/VulkanSDK-Installer.exe -L "https://sdk.lunarg.com/sdk/download/${env:VULKAN_VERSION}/windows/vulkansdk-windows-X64-${env:VULKAN_VERSION}.exe"
- & "$env:RUNNER_TEMP\VulkanSDK-Installer.exe" --accept-licenses --default-answer --confirm-command install
- Add-Content $env:GITHUB_ENV "VULKAN_SDK=C:\VulkanSDK\${env:VULKAN_VERSION}"
- Add-Content $env:GITHUB_PATH "C:\VulkanSDK\${env:VULKAN_VERSION}\bin"
-
- name: Install Ninja
id: install_ninja
run: |
diff --git a/.github/workflows/build-rpc.yml b/.github/workflows/build-rpc.yml
deleted file mode 100644
index d04dc375b5..0000000000
--- a/.github/workflows/build-rpc.yml
+++ /dev/null
@@ -1,66 +0,0 @@
-name: CI (rpc)
-
-on:
- workflow_dispatch: # allows manual triggering
- push:
- branches:
- - master
- paths: [
- '.github/workflows/build-rpc.yml',
- '**/CMakeLists.txt',
- '**/.cmake',
- '**/*.h',
- '**/*.hpp',
- '**/*.c',
- '**/*.cpp'
- ]
-
- pull_request:
- types: [opened, synchronize, reopened]
- paths: [
- '.github/workflows/build-rpc.yml',
- 'ggml/src/ggml-rpc/**'
- ]
-
-concurrency:
- group: ${{ github.workflow }}-${{ github.head_ref && github.ref || github.run_id }}
- cancel-in-progress: true
-
-env:
- GGML_NLOOP: 3
- GGML_N_THREADS: 1
- LLAMA_ARG_LOG_COLORS: 1
- LLAMA_ARG_LOG_PREFIX: 1
- LLAMA_ARG_LOG_TIMESTAMPS: 1
-
-jobs:
- ubuntu-24-rpc:
- runs-on: ${{ 'ubuntu-24.04-arm' || 'ubuntu-24.04' }}
-
- continue-on-error: true
-
- steps:
- - name: Clone
- id: checkout
- uses: actions/checkout@v6
-
- - name: Dependencies
- id: depends
- run: |
- sudo apt-get update
- sudo apt-get install build-essential libssl-dev ninja-build
-
- - name: Build
- id: cmake_build
- run: |
- cmake -B build \
- -G "Ninja" \
- -DCMAKE_BUILD_TYPE=Release \
- -DGGML_RPC=ON
- time cmake --build build --config Release -j $(nproc)
-
- - name: Test
- id: cmake_test
- run: |
- cd build
- ctest -L main --verbose
diff --git a/.github/workflows/build-vulkan.yml b/.github/workflows/build-vulkan.yml
index 01113803ff..15ff4b0af3 100644
--- a/.github/workflows/build-vulkan.yml
+++ b/.github/workflows/build-vulkan.yml
@@ -93,19 +93,13 @@ jobs:
run: |
echo "VULKAN_SDK_VERSION=$(curl https://vulkan.lunarg.com/sdk/latest/linux.txt)" >> "$GITHUB_ENV"
- - name: Use Vulkan SDK Cache
- uses: actions/cache@v5
- id: cache-sdk
- with:
- path: ./vulkan_sdk
- key: cache-gha-vulkan-sdk-${{ env.VULKAN_SDK_VERSION }}-${{ runner.os }}
-
- name: Setup Vulkan SDK
- if: steps.cache-sdk.outputs.cache-hit != 'true'
- uses: ./.github/actions/linux-setup-vulkan
+ id: setup
+ uses: ./.github/actions/unarchive-tar
with:
+ url: https://sdk.lunarg.com/sdk/download/${{ env.VULKAN_SDK_VERSION }}/linux/vulkan_sdk.tar.xz
path: ./vulkan_sdk
- version: ${{ env.VULKAN_SDK_VERSION }}
+ strip: 1
- name: ccache
uses: ggml-org/[email protected]
@@ -133,3 +127,56 @@ jobs:
# This is using llvmpipe and runs slower than other backends
# test-backend-ops is too slow on llvmpipe, skip it
ctest -L main -E test-backend-ops --verbose --timeout 900
+
+ windows:
+ runs-on: windows-2025
+
+ env:
+ VULKAN_VERSION: 1.4.357.0
+
+ steps:
+ - name: Clone
+ id: checkout
+ uses: actions/checkout@v6
+
+ - name: ccache
+ uses: ggml-org/[email protected]
+ with:
+ key: cpu-windows-2025-x64-vulkan
+ variant: ccache
+ evict-old-files: 1d
+ save: ${{ github.event_name == 'push' && github.ref == 'refs/heads/master' }}
+
+ - name: Install Vulkan SDK
+ id: get_vulkan
+ run: |
+ curl.exe -o $env:RUNNER_TEMP/VulkanSDK-Installer.exe -L "https://sdk.lunarg.com/sdk/download/${env:VULKAN_VERSION}/windows/vulkansdk-windows-X64-${env:VULKAN_VERSION}.exe"
+ & "$env:RUNNER_TEMP\VulkanSDK-Installer.exe" --accept-licenses --default-answer --confirm-command install
+ Add-Content $env:GITHUB_ENV "VULKAN_SDK=C:\VulkanSDK\${env:VULKAN_VERSION}"
+ Add-Content $env:GITHUB_PATH "C:\VulkanSDK\${env:VULKAN_VERSION}\bin"
+
+ - name: Install Ninja
+ id: install_ninja
+ run: |
+ choco install ninja
+
+ - name: Build
+ id: cmake_build
+ run: |
+ cmake -S . -B build -G "Ninja Multi-Config" `
+ -D CMAKE_TOOLCHAIN_FILE=cmake/x64-windows-llvm.cmake `
+ -DCMAKE_BUILD_TYPE=Release `
+ -DGGML_NATIVE=OFF `
+ -DLLAMA_BUILD_SERVER=ON `
+ -DGGML_RPC=ON `
+ -DGGML_BACKEND_DL=ON `
+ -DGGML_CPU_ALL_VARIANTS=ON `
+ -DGGML_VULKAN=ON `
+ -DLLAMA_BUILD_BORINGSSL=ON
+ cmake --build build --config Release -j ${env:NUMBER_OF_PROCESSORS}
+
+ - name: Test
+ id: cmake_test
+ run: |
+ cd build
+ ctest -L main -C Release --verbose --timeout 900