LLM inference in C/C++
C++ ggml
ci : add models backend check (#29651)
Sigbjørn Skjæret72db1e02ff0d
diff --git a/.github/workflows/fusion.yml b/.github/workflows/fusion.ymldeleted file mode 100644index 7c8596467a..0000000000--- a/.github/workflows/fusion.yml+++ /dev/null@@ -1,71 +0,0 @@-name: Fusion--on:- workflow_dispatch: # allows manual triggering- push:- branches:- - master- paths: [- '.github/workflows/fusion.yml',- 'ggml/**',- 'tests/fusion/**',- 'tests/test-fusion.cpp',- 'tests/test-llama-archs.cpp',- 'src/models/**'- ]-- pull_request:- types: [opened, synchronize, reopened]- paths: [- '.github/workflows/fusion.yml',- 'ggml/**',- 'tests/fusion/**',- 'tests/test-fusion.cpp',- 'tests/test-llama-archs.cpp',- 'src/models/**'- ]--concurrency:- group: ${{ github.workflow }}-${{ github.head_ref && github.ref || github.run_id }}- cancel-in-progress: true--env:- GGML_NLOOP: 3- GGML_N_THREADS: 1- LLAMA_ARG_LOG_COLORS: 1- LLAMA_ARG_LOG_PREFIX: 1- LLAMA_ARG_LOG_TIMESTAMPS: 1--jobs:- # TODO: add jobs for other backends as they adopt the fusion debug API- metal:- runs-on: [self-hosted, macOS, ARM64]-- steps:- - name: Clone- id: checkout- uses: actions/checkout@v6-- - name: Build- id: cmake_build- run: |- cmake -B build \- -DCMAKE_BUILD_TYPE=Release \- -DLLAMA_FATAL_WARNINGS=ON \- -DLLAMA_OPENSSL=OFF \- -DGGML_SCHED_NO_REALLOC=ON \- -DGGML_BLAS=OFF \- -DGGML_METAL=ON- time cmake --build build --config Release --target test-llama-archs -j $(sysctl -n hw.logicalcpu)- time cmake --build build --config Release --target test-fusion -j $(sysctl -n hw.logicalcpu)-- - name: Generate models- id: generate_models- run: |- rm -rf build-ci-models && mkdir -p build-ci-models- ./build/bin/test-llama-archs -o build-ci-models-- - name: Test fusion- id: test_fusion- run: |- ./build/bin/test-fusion --models build-ci-models --device MTL0 --check tests/fusion/MTL.csvdiff --git a/.github/workflows/models-check.yml b/.github/workflows/models-check.ymlnew file mode 100644index 0000000000..51563eacee--- /dev/null+++ b/.github/workflows/models-check.yml@@ -0,0 +1,445 @@+name: Models Backend Check++on:+ workflow_dispatch: # allows manual triggering+ push:+ branches:+ - master+ paths: [+ '.github/workflows/models-check.yml',+ 'ggml/**',+ 'tests/fusion/**',+ 'tests/test-fusion.cpp',+ 'tests/test-llama-archs.cpp',+ 'src/llama-graph.cpp',+ 'src/llama-model*',+ 'src/models/**'+ ]++ pull_request:+ types: [opened, synchronize, reopened]+ paths: [+ '.github/workflows/models-check.yml',+ 'ggml/**',+ 'tests/fusion/**',+ 'tests/test-fusion.cpp',+ 'tests/test-llama-archs.cpp',+ 'src/llama-graph.cpp',+ 'src/llama-model*',+ 'src/models/**'+ ]++concurrency:+ group: ${{ github.workflow }}-${{ github.head_ref && github.ref || github.run_id }}+ cancel-in-progress: true++env:+ # note: this is dud token to avoid rate limiting (https://github.com/ggml-org/llama.cpp/pull/25706#issuecomment-4979941302)+ HF_TOKEN: ${{ secrets.HF_TOKEN_CI }}+ GGML_NLOOP: 3+ GGML_N_THREADS: 1+ LLAMA_ARG_LOG_COLORS: 1+ LLAMA_ARG_LOG_PREFIX: 1+ LLAMA_ARG_LOG_TIMESTAMPS: 1++jobs:+ cuda:+ runs-on: "hf-jobs-t4-medium:cuda13"++ steps:+ - name: Clone+ id: checkout+ uses: actions/checkout@v6++ - name: Install dependencies+ run: |+ sudo apt update+ sudo apt install -y cmake time python3 python3-venv python3-pip++ - name: ccache+ uses: ggml-org/[email protected]+ with:+ restore: false+ save: false++ - name: ccache-buckets-restore+ uses: ./.github/actions/ccache-buckets+ with:+ key: models-check-cuda+ folder: llama.cpp+ hf_bucket: ggml-org/cache++ - name: Build+ id: cmake_build+ run: |+ cmake -B build \+ -DCMAKE_BUILD_TYPE=Release \+ -DLLAMA_FATAL_WARNINGS=ON \+ -DLLAMA_OPENSSL=OFF \+ -DGGML_SCHED_NO_REALLOC=ON \+ -DCMAKE_CUDA_COMPILER=/usr/local/cuda/bin/nvcc \+ -DGGML_CUDA=ON+ time cmake --build build --config Release --target test-llama-archs -j$(nproc)+ time cmake --build build --config Release --target test-fusion -j$(nproc)++ - name: ccache-buckets-save+ if: ${{ github.event_name == 'push' && github.ref == 'refs/heads/master' }}+ uses: ./.github/actions/ccache-buckets+ env:+ HF_TOKEN: ${{ secrets.HF_TOKEN_CACHE_OUTPUT }}+ with:+ key: models-check-cuda+ folder: llama.cpp+ evict-old-files: 1d+ hf_bucket: ggml-org/cache+ save: true++ # - name: Generate models+ # id: generate_models+ # run: |+ # rm -rf build-ci-models && mkdir -p build-ci-models+ # ./build/bin/test-llama-archs -o build-ci-models++ # TODO: add for backends as they adopt the fusion debug API+ # - name: Test fusion+ # id: test_fusion+ # run: |+ # ./build/bin/test-fusion --models build-ci-models --device CUDA0 --check tests/fusion/CUDA.csv++ - name: Test archs+ id: test_archs+ run: |+ GGML_CUDA_DEVICES=1 ./build/bin/test-llama-archs -s 1+ GGML_CUDA_DEVICES=2 ./build/bin/test-llama-archs -s 1+ GGML_CUDA_DEVICES=3 ./build/bin/test-llama-archs -s 1+ GGML_CUDA_DEVICES=4 ./build/bin/test-llama-archs -s 1++ metal:+ runs-on: [self-hosted, macOS, ARM64]++ steps:+ - name: Clone+ id: checkout+ uses: actions/checkout@v6++ - name: Build+ id: cmake_build+ run: |+ cmake -B build \+ -DCMAKE_BUILD_TYPE=Release \+ -DLLAMA_FATAL_WARNINGS=ON \+ -DLLAMA_OPENSSL=OFF \+ -DGGML_SCHED_NO_REALLOC=ON \+ -DGGML_BLAS=OFF \+ -DGGML_METAL=ON+ time cmake --build build --config Release --target test-llama-archs -j $(sysctl -n hw.logicalcpu)+ time cmake --build build --config Release --target test-fusion -j $(sysctl -n hw.logicalcpu)++ - name: Generate models+ id: generate_models+ run: |+ rm -rf build-ci-models && mkdir -p build-ci-models+ ./build/bin/test-llama-archs -o build-ci-models++ - name: Test fusion+ id: test_fusion+ run: |+ ./build/bin/test-fusion --models build-ci-models --device MTL0 --check tests/fusion/MTL.csv++ - name: Test archs+ id: test_archs+ run: |+ GGML_METAL_DEVICES=1 ./build/bin/test-llama-archs -s 1+ GGML_METAL_DEVICES=2 ./build/bin/test-llama-archs -s 1+ GGML_METAL_DEVICES=3 ./build/bin/test-llama-archs -s 1+ GGML_METAL_DEVICES=4 ./build/bin/test-llama-archs -s 1++ rocm:+ runs-on: [self-hosted, Linux, gfx1201]+ container: "rocm/dev-ubuntu-24.04:7.2.4-complete"++ steps:+ - name: Clone+ id: checkout+ uses: actions/checkout@v6++ - name: Install dependencies+ run: |+ apt update+ apt install -y build-essential jq cmake time python3 python3-venv python3-pip++ - name: ccache+ uses: ggml-org/[email protected]+ with:+ restore: false+ save: false++ - name: ccache-buckets-restore+ uses: ./.github/actions/ccache-buckets+ with:+ key: models-check-rocm+ folder: llama.cpp+ hf_bucket: ggml-org/cache++ - name: Build+ id: cmake_build+ run: |+ cmake -B build \+ -DCMAKE_BUILD_TYPE=Release \+ -DLLAMA_FATAL_WARNINGS=ON \+ -DLLAMA_OPENSSL=OFF \+ -DGGML_SCHED_NO_REALLOC=ON \+ -DCMAKE_HIP_COMPILER=$(hipconfig -l)/clang \+ -DGPU_TARGETS=gfx1201 \+ -DGGML_HIP=ON+ time cmake --build build --config Release --target test-llama-archs -j$(nproc)+ time cmake --build build --config Release --target test-fusion -j$(nproc)++ - name: ccache-buckets-save+ if: ${{ github.event_name == 'push' && github.ref == 'refs/heads/master' }}+ uses: ./.github/actions/ccache-buckets+ env:+ HF_TOKEN: ${{ secrets.HF_TOKEN_CACHE_OUTPUT }}+ with:+ key: models-check-rocm+ folder: llama.cpp+ evict-old-files: 1d+ hf_bucket: ggml-org/cache+ save: true++ # - name: Generate models+ # id: generate_models+ # run: |+ # rm -rf build-ci-models && mkdir -p build-ci-models+ # ./build/bin/test-llama-archs -o build-ci-models++ # TODO: add for backends as they adopt the fusion debug API+ # - name: Test fusion+ # id: test_fusion+ # run: |+ # ./build/bin/test-fusion --models build-ci-models --device CUDA0 --check tests/fusion/CUDA.csv++ - name: Test archs+ id: test_archs+ run: |+ GGML_CUDA_DEVICES=1 ./build/bin/test-llama-archs -s 1+ GGML_CUDA_DEVICES=2 ./build/bin/test-llama-archs -s 1+ GGML_CUDA_DEVICES=3 ./build/bin/test-llama-archs -s 1+ GGML_CUDA_DEVICES=4 ./build/bin/test-llama-archs -s 1++ vulkan-nvidia:+ runs-on: "hf-jobs-t4-small:ubuntu26_04"++ steps:+ - name: Clone+ id: checkout+ uses: actions/checkout@v6++ - name: Install dependencies+ run: |+ sudo apt update+ sudo apt install -y build-essential cmake libxcb-xinput0 libxcb-xinerama0 libxcb-cursor-dev libvulkan-dev glslc spirv-headers vulkan-tools mesa-vulkan-drivers libglvnd0 libgl1 libglx0 libegl1 libgles2 time python3 python3-venv python3-pip++ - name: ccache+ uses: ggml-org/[email protected]+ with:+ restore: false+ save: false++ - name: ccache-buckets-restore+ uses: ./.github/actions/ccache-buckets+ with:+ key: models-check-vulkan-nvidia+ folder: llama.cpp+ hf_bucket: ggml-org/cache++ - name: Build+ id: cmake_build+ run: |+ cmake -B build \+ -DCMAKE_BUILD_TYPE=Release \+ -DLLAMA_FATAL_WARNINGS=ON \+ -DLLAMA_OPENSSL=OFF \+ -DGGML_SCHED_NO_REALLOC=ON \+ -DGGML_VULKAN=ON+ time cmake --build build --config Release --target test-llama-archs -j$(nproc)+ time cmake --build build --config Release --target test-fusion -j$(nproc)++ - name: ccache-buckets-save+ if: ${{ github.event_name == 'push' && github.ref == 'refs/heads/master' }}+ uses: ./.github/actions/ccache-buckets+ env:+ HF_TOKEN: ${{ secrets.HF_TOKEN_CACHE_OUTPUT }}+ with:+ key: models-check-vulkan-nvidia+ folder: llama.cpp+ evict-old-files: 1d+ hf_bucket: ggml-org/cache+ save: true++ # - name: Generate models+ # id: generate_models+ # run: |+ # rm -rf build-ci-models && mkdir -p build-ci-models+ # ./build/bin/test-llama-archs -o build-ci-models++ # TODO: add for backends as they adopt the fusion debug API+ # - name: Test fusion+ # id: test_fusion+ # run: |+ # ./build/bin/test-fusion --models build-ci-models --device Vulkan0 --check tests/fusion/Vulkan.csv++ - name: Test archs+ id: test_archs+ run: |+ ./build/bin/test-llama-archs -s 1++ vulkan-amd:+ runs-on: [self-hosted, Linux, gfx1201]+ container: "ubuntu:26.04"++ steps:+ - name: Clone+ id: checkout+ uses: actions/checkout@v6++ - name: Install dependencies+ run: |+ apt update+ apt install -y build-essential jq cmake libxcb-xinput0 libxcb-xinerama0 libxcb-cursor-dev libvulkan-dev glslc spirv-headers vulkan-tools mesa-vulkan-drivers libglvnd0 libgl1 libglx0 libegl1 libgles2 time python3 python3-venv python3-pip++ - name: ccache+ uses: ggml-org/[email protected]+ with:+ restore: false+ save: false++ - name: ccache-buckets-restore+ uses: ./.github/actions/ccache-buckets+ with:+ key: models-check-vulkan-amd+ folder: llama.cpp+ hf_bucket: ggml-org/cache++ - name: Build+ id: cmake_build+ run: |+ cmake -B build \+ -DCMAKE_BUILD_TYPE=Release \+ -DLLAMA_FATAL_WARNINGS=ON \+ -DLLAMA_OPENSSL=OFF \+ -DGGML_SCHED_NO_REALLOC=ON \+ -DGGML_VULKAN=ON+ time cmake --build build --config Release --target test-llama-archs -j$(nproc)+ time cmake --build build --config Release --target test-fusion -j$(nproc)++ - name: ccache-buckets-save+ if: ${{ github.event_name == 'push' && github.ref == 'refs/heads/master' }}+ uses: ./.github/actions/ccache-buckets+ env:+ HF_TOKEN: ${{ secrets.HF_TOKEN_CACHE_OUTPUT }}+ with:+ key: models-check-vulkan-amd+ folder: llama.cpp+ evict-old-files: 1d+ hf_bucket: ggml-org/cache+ save: true++ # - name: Generate models+ # id: generate_models+ # run: |+ # rm -rf build-ci-models && mkdir -p build-ci-models+ # ./build/bin/test-llama-archs -o build-ci-models++ # TODO: add for backends as they adopt the fusion debug API+ # - name: Test fusion+ # id: test_fusion+ # run: |+ # ./build/bin/test-fusion --models build-ci-models --device Vulkan0 --check tests/fusion/Vulkan.csv++ - name: Test archs+ id: test_archs+ run: |+ ./build/bin/test-llama-archs -s 1++ webgpu-nvidia:+ runs-on: "hf-jobs-t4-small:ubuntu26_04"++ steps:+ - name: Clone+ id: checkout+ uses: actions/checkout@v6++ - name: Install dependencies+ run: |+ sudo apt update+ sudo apt install -y build-essential cmake libxcb-xinput0 libxcb-xinerama0 libxcb-cursor-dev libvulkan1 mesa-vulkan-drivers libglvnd0 libgl1 libglx0 libegl1 libgles2 time python3 python3-venv python3-pip++ - name: ccache+ uses: ggml-org/[email protected]+ with:+ restore: false+ save: false++ - name: ccache-buckets-restore+ uses: ./.github/actions/ccache-buckets+ with:+ key: models-check-webgpu-nvidia+ folder: llama.cpp+ hf_bucket: ggml-org/cache++ - name: Dawn Dependency+ id: dawn-depends+ run: |+ DAWN_VERSION="v20260908.214631"+ DAWN_OWNER="google"+ DAWN_REPO="dawn"+ DAWN_ASSET_NAME="Dawn-94c3c9cc0d5fb2e85aebb370fa8d37b71aa34655-ubuntu-latest-Release"+ echo "Fetching release asset from https://github.com/google/dawn/releases/download/${DAWN_VERSION}/${DAWN_ASSET_NAME}.tar.gz"+ curl -L -o artifact.tar.gz \+ "https://github.com/google/dawn/releases/download/${DAWN_VERSION}/${DAWN_ASSET_NAME}.tar.gz"+ mkdir dawn+ tar -xvf artifact.tar.gz -C dawn --strip-components=1++ - name: Build+ id: cmake_build+ run: |+ cmake -B build \+ -DCMAKE_BUILD_TYPE=Release \+ -DLLAMA_FATAL_WARNINGS=ON \+ -DLLAMA_OPENSSL=OFF \+ -DGGML_SCHED_NO_REALLOC=ON \+ -DCMAKE_PREFIX_PATH="$GITHUB_WORKSPACE/dawn" \+ -DDawn_DIR="$GITHUB_WORKSPACE/dawn/lib64/cmake/Dawn" \+ -DGGML_WEBGPU=ON+ time cmake --build build --config Release --target test-llama-archs -j$(nproc)+ time cmake --build build --config Release --target test-fusion -j$(nproc)++ - name: ccache-buckets-save+ if: ${{ github.event_name == 'push' && github.ref == 'refs/heads/master' }}+ uses: ./.github/actions/ccache-buckets+ env:+ HF_TOKEN: ${{ secrets.HF_TOKEN_CACHE_OUTPUT }}+ with:+ key: models-check-webgpu-nvidia+ folder: llama.cpp+ evict-old-files: 1d+ hf_bucket: ggml-org/cache+ save: true++ # - name: Generate models+ # id: generate_models+ # run: |+ # rm -rf build-ci-models && mkdir -p build-ci-models+ # ./build/bin/test-llama-archs -o build-ci-models++ # TODO: add for backends as they adopt the fusion debug API+ # - name: Test fusion+ # id: test_fusion+ # run: |+ # ./build/bin/test-fusion --models build-ci-models --device WebGPU --check tests/fusion/WebGPU.csv++ - name: Test archs+ id: test_archs+ run: |+ ./build/bin/test-llama-archs -s 1