LLM inference in C/C++
C++ ggml
ci : apply ccache-clear with older/min/dry-run to all ccache jobs (#27602)
Georgi Gerganov985b14912bc9
diff --git a/.github/actions/ccache-clear/action.yml b/.github/actions/ccache-clear/action.ymlindex 2420045edf..fc5da4f6ed 100644--- a/.github/actions/ccache-clear/action.yml+++ b/.github/actions/ccache-clear/action.yml@@ -21,68 +21,30 @@ inputs:runs:using: "composite"steps:- - name: Clear caches+ - name: Install GitHub CLI if missingshell: bash- env:- CLEAR_KEY: ${{ inputs.key }}- CLEAR_OLDER: ${{ inputs.older }}- CLEAR_MIN: ${{ inputs.min }}- CLEAR_DRY_RUN: ${{ inputs.dry-run }}run: |- # Convert a duration (e.g. 90m, 1h, 1d, plain seconds) to seconds- to_seconds() {- local val="$1"- [[ "$val" =~ ^[0-9]+$ ]] && { echo "$val"; return 0; }- local num="${val%?}" unit="${val: -1}" mult- [[ "$num" =~ ^[0-9]+$ ]] || return 1- case "$unit" in- s) mult=1 ;;- m) mult=60 ;;- h) mult=3600 ;;- d) mult=86400 ;;- *) return 1 ;;- esac- echo $((num * mult))- }-- [[ "$CLEAR_MIN" =~ ^[0-9]+$ ]] || { echo "Invalid min value: $CLEAR_MIN" >&2; exit 1; }- [[ "$CLEAR_DRY_RUN" =~ ^(true|false)$ ]] || { echo "Invalid dry-run value: $CLEAR_DRY_RUN" >&2; exit 1; }-- CACHES=$(gh cache list --key "ccache-$CLEAR_KEY" --json id,key,createdAt --jq '.[] | [.createdAt, .id, .key] | @tsv' 2>/dev/null | LC_ALL=C sort)- if [ -z "$CACHES" ]; then- echo "No caches found with key prefix: $CLEAR_KEY"- exit 0- fi-- TOTAL=$(( $(wc -l <<< "$CACHES") ))-- echo "Found $TOTAL cache(s) with key prefix: $CLEAR_KEY (oldest first):"- while IFS=$'\t' read -r CREATED ID KEY; do- printf ' %s %s %s\n' "$CREATED" "$ID" "$KEY"- done <<< "$CACHES"-- CUTOFF=""- if [ -n "$CLEAR_OLDER" ]; then- OLDER_SECONDS=$(to_seconds "$CLEAR_OLDER") || { echo "Invalid older value: $CLEAR_OLDER (expected e.g. 90m, 1h, 1d)" >&2; exit 1; }- CUTOFF=$(( $(date +%s) - OLDER_SECONDS ))+ # e.g. in container jobs, where it is not preinstalled+ if ! command -v gh >/dev/null 2>&1; then+ echo "GitHub CLI not found, installing..."+ if ! command -v curl >/dev/null 2>&1; then+ apt-get update >/dev/null 2>&1 || true+ apt-get install -y curl >/dev/null 2>&1 || true+ fi+ mkdir -p -m 755 /etc/apt/keyrings+ curl -fsSL https://cli.github.com/packages/githubcli-archive-keyring.gpg | tee /etc/apt/keyrings/githubcli-archive-keyring.gpg >/dev/null+ chmod go+r /etc/apt/keyrings/githubcli-archive-keyring.gpg+ echo "deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/githubcli-archive-keyring.gpg] https://cli.github.com/packages stable main" > /etc/apt/sources.list.d/github-cli.list+ apt-get update >/dev/null 2>&1 || true+ apt-get install -y gh || { echo "Failed to install GitHub CLI (gh)" >&2; exit 1; }fi+ command -v gh >/dev/null 2>&1 || { echo "GitHub CLI (gh) is required but could not be installed" >&2; exit 1; }- # Caches are sorted oldest first- DELETED=0- while IFS=$'\t' read -r CREATED ID KEY; do- if [ -n "$CUTOFF" ] && [ "$(date -d "$CREATED" +%s)" -ge "$CUTOFF" ]; then- echo "Rest are not older than $CLEAR_OLDER, stopping"- break- fi- if [ $((TOTAL - DELETED - 1)) -lt "$CLEAR_MIN" ]; then- echo "Keeping at least $CLEAR_MIN cache(s), stopping"- break- fi- if [ "$CLEAR_DRY_RUN" = "true" ]; then- echo "Would delete cache: $ID ($KEY)"- else- echo "Deleting cache: $ID ($KEY)"- gh cache delete "$ID"- fi- DELETED=$((DELETED + 1))- done <<< "$CACHES"+ - name: Clear caches+ shell: bash+ run: |+ bash scripts/ccache-clear.sh \+ --key "${{ inputs.key }}" \+ --older "${{ inputs.older }}" \+ --min "${{ inputs.min }}" \+ ${{ inputs.dry-run == 'true' && '--dry-run' || '' }}diff --git a/.github/workflows/build-apple.yml b/.github/workflows/build-apple.ymlindex 289e5144e6..9a4a691d52 100644--- a/.github/workflows/build-apple.yml+++ b/.github/workflows/build-apple.yml@@ -73,6 +73,16 @@ jobs:cd buildctest -L main -E "test-llama-archs" --verbose --timeout 900+ - name: ccache-clear+ uses: ./.github/actions/ccache-clear+ env:+ GH_TOKEN: ${{ github.token }}+ with:+ key: apple-arm64+ older: 5m+ min: 1+ dry-run: ${{ github.event_name != 'push' || github.ref != 'refs/heads/master' }}+macos-latest-x64:runs-on: macos-15-intel@@ -109,6 +119,16 @@ jobs:cd buildctest -L main --verbose --timeout 900+ - name: ccache-clear+ uses: ./.github/actions/ccache-clear+ env:+ GH_TOKEN: ${{ github.token }}+ with:+ key: apple-x64+ older: 5m+ min: 1+ dry-run: ${{ github.event_name != 'push' || github.ref != 'refs/heads/master' }}+macos-latest-ios-xcode:runs-on: macos-latest@@ -163,14 +183,6 @@ jobs:id: checkoutuses: actions/checkout@v6- # TODO: this likely does not do anything - if yes, remove it- - name: ccache- uses: ggml-org/[email protected]- with:- key: apple-tvos- evict-old-files: 1d- save: ${{ github.event_name == 'push' && github.ref == 'refs/heads/master' }}-- name: Buildid: cmake_buildrun: |@@ -196,14 +208,6 @@ jobs:id: checkoutuses: actions/checkout@v6- # TODO: this likely does not do anything - if yes, remove it- - name: ccache- uses: ggml-org/[email protected]- with:- key: apple-visionos- evict-old-files: 1d- save: ${{ github.event_name == 'push' && github.ref == 'refs/heads/master' }}-- name: Buildid: cmake_buildrun: |@@ -234,14 +238,6 @@ jobs:id: checkoutuses: actions/checkout@v6- # TODO: this likely does not do anything - if yes, remove it- - name: ccache- uses: ggml-org/[email protected]- with:- key: apple-swift- evict-old-files: 1d- save: ${{ github.event_name == 'push' && github.ref == 'refs/heads/master' }}-- name: Download xcframework artifactuses: actions/download-artifact@v7with:diff --git a/.github/workflows/build-cpu.yml b/.github/workflows/build-cpu.ymlindex f39304cabf..b62fe55d6c 100644--- a/.github/workflows/build-cpu.yml+++ b/.github/workflows/build-cpu.yml@@ -125,7 +125,7 @@ jobs:GH_TOKEN: ${{ github.token }}with:key: cpu-${{ matrix.os }}- older: 1h+ older: 5mmin: 1dry-run: ${{ github.event_name != 'push' || github.ref != 'refs/heads/master' }}@@ -215,3 +215,13 @@ jobs:# cd build# $env:LLAMA_SKIP_TESTS_SLOW_ON_EMULATOR = 1# & $sde -future -- ctest -L main -C Release --verbose --timeout 900++ - name: ccache-clear+ uses: ./.github/actions/ccache-clear+ env:+ GH_TOKEN: ${{ github.token }}+ with:+ key: cpu-windows-2025-${{ matrix.build }}+ older: 5m+ min: 1+ dry-run: ${{ github.event_name != 'push' || github.ref != 'refs/heads/master' }}diff --git a/.github/workflows/build-cuda-ubuntu.yml b/.github/workflows/build-cuda-ubuntu.ymlindex 2528b18573..ed5164756a 100644--- a/.github/workflows/build-cuda-ubuntu.yml+++ b/.github/workflows/build-cuda-ubuntu.yml@@ -72,6 +72,16 @@ jobs:-DGGML_CUDA_CUB_3DOT2=ONcmake --build build+ - name: ccache-clear+ uses: ./.github/actions/ccache-clear+ env:+ GH_TOKEN: ${{ github.token }}+ with:+ key: cuda-ubuntu-24.04-cuda+ older: 5m+ min: 1+ dry-run: ${{ github.event_name != 'push' || github.ref != 'refs/heads/master' }}+hip:runs-on: ubuntu-22.04container: rocm/dev-ubuntu-22.04:6.1.2@@ -103,6 +113,16 @@ jobs:-DGGML_HIP=ONcmake --build build --config Release -j $(nproc)+ - name: ccache-clear+ uses: ./.github/actions/ccache-clear+ env:+ GH_TOKEN: ${{ github.token }}+ with:+ key: cuda-ubuntu-22.04-hip+ older: 5m+ min: 1+ dry-run: ${{ github.event_name != 'push' || github.ref != 'refs/heads/master' }}+musa:runs-on: ubuntu-22.04container: mthreads/musa:rc4.3.0-devel-ubuntu22.04-amd64@@ -131,3 +151,13 @@ jobs:cmake -B build -S . \-DGGML_MUSA=ONtime cmake --build build --config Release -j $(nproc)++ - name: ccache-clear+ uses: ./.github/actions/ccache-clear+ env:+ GH_TOKEN: ${{ github.token }}+ with:+ key: cuda-ubuntu-22.04-musa+ older: 5m+ min: 1+ dry-run: ${{ github.event_name != 'push' || github.ref != 'refs/heads/master' }}diff --git a/.github/workflows/build-opencl.yml b/.github/workflows/build-opencl.ymlindex 251b1f8d59..c0adc7e496 100644--- a/.github/workflows/build-opencl.yml+++ b/.github/workflows/build-opencl.yml@@ -80,3 +80,13 @@ jobs:run: |cmake -S . -B build -G "Ninja Multi-Config" -D CMAKE_TOOLCHAIN_FILE=cmake/arm64-windows-llvm.cmake -DCMAKE_PREFIX_PATH="$env:RUNNER_TEMP/opencl-arm64-release" -DGGML_OPENCL=ON -DGGML_OPENCL_USE_ADRENO_KERNELS=ON -DLLAMA_BUILD_BORINGSSL=ONcmake --build build --config Release -j ${env:NUMBER_OF_PROCESSORS}++ - name: ccache-clear+ uses: ./.github/actions/ccache-clear+ env:+ GH_TOKEN: ${{ github.token }}+ with:+ key: opencl-windows-2025-x64+ older: 5m+ min: 1+ dry-run: ${{ github.event_name != 'push' || github.ref != 'refs/heads/master' }}diff --git a/.github/workflows/build-openvino.yml b/.github/workflows/build-openvino.ymlindex ee4268f970..0316e7ad97 100644--- a/.github/workflows/build-openvino.yml+++ b/.github/workflows/build-openvino.yml@@ -167,3 +167,13 @@ jobs:cd buildctest --test-dir ReleaseOV -L main -E "test-llama-archs|test-recurrent-state-rollback-nemotron-h" -C Release --verbose --timeout 3000++ - name: ccache-clear+ uses: ./.github/actions/ccache-clear+ env:+ GH_TOKEN: ${{ github.token }}+ with:+ key: openvino-windows-2022+ older: 5m+ min: 1+ dry-run: ${{ github.event_name != 'push' || github.ref != 'refs/heads/master' }}diff --git a/.github/workflows/build-sycl.yml b/.github/workflows/build-sycl.ymlindex deb0e5479a..7beac81772 100644--- a/.github/workflows/build-sycl.yml+++ b/.github/workflows/build-sycl.yml@@ -96,6 +96,16 @@ jobs:-DGGML_SYCL_F16=${{ matrix.fp16 }}time cmake --build build --config Release -j $(nproc)+ - name: ccache-clear+ uses: ./.github/actions/ccache-clear+ env:+ GH_TOKEN: ${{ github.token }}+ with:+ key: sycl-ubuntu-24-${{ matrix.build }}+ older: 5m+ min: 1+ dry-run: ${{ github.event_name != 'push' || github.ref != 'refs/heads/master' }}+windows-latest-sycl:runs-on: windows-2022@@ -139,3 +149,13 @@ jobs:- name: Buildid: cmake_buildrun: examples/sycl/win-build-sycl.bat++ - name: ccache-clear+ uses: ./.github/actions/ccache-clear+ env:+ GH_TOKEN: ${{ github.token }}+ with:+ key: sycl-windows-latest+ older: 5m+ min: 1+ dry-run: ${{ github.event_name != 'push' || github.ref != 'refs/heads/master' }}diff --git a/.github/workflows/build-vulkan.yml b/.github/workflows/build-vulkan.ymlindex 15ff4b0af3..74d1c69368 100644--- a/.github/workflows/build-vulkan.yml+++ b/.github/workflows/build-vulkan.yml@@ -55,7 +55,7 @@ jobs:- name: ccacheuses: ggml-org/[email protected]with:- key: vulkan-ubuntu-24.04-arm-new+ key: vulkan-ubuntu-24.04-armvariant: ccacheevict-old-files: 1dsave: ${{ github.event_name == 'push' && github.ref == 'refs/heads/master' }}@@ -73,6 +73,16 @@ jobs:run: |time cmake --build build -j $(nproc)+ - name: ccache-clear+ uses: ./.github/actions/ccache-clear+ env:+ GH_TOKEN: ${{ github.token }}+ with:+ key: vulkan-ubuntu-24.04-arm+ older: 5m+ min: 1+ dry-run: ${{ github.event_name != 'push' || github.ref != 'refs/heads/master' }}+ubuntu-llvmpipe:runs-on: ubuntu-24.04@@ -128,6 +138,16 @@ jobs:# test-backend-ops is too slow on llvmpipe, skip itctest -L main -E test-backend-ops --verbose --timeout 900+ - name: ccache-clear+ uses: ./.github/actions/ccache-clear+ env:+ GH_TOKEN: ${{ github.token }}+ with:+ key: vulkan-ubuntu-24.04-llvmpipe+ older: 5m+ min: 1+ dry-run: ${{ github.event_name != 'push' || github.ref != 'refs/heads/master' }}+windows:runs-on: windows-2025@@ -180,3 +200,13 @@ jobs:run: |cd buildctest -L main -C Release --verbose --timeout 900++ - name: ccache-clear+ uses: ./.github/actions/ccache-clear+ env:+ GH_TOKEN: ${{ github.token }}+ with:+ key: cpu-windows-2025-x64-vulkan+ older: 5m+ min: 1+ dry-run: ${{ github.event_name != 'push' || github.ref != 'refs/heads/master' }}diff --git a/.github/workflows/build-wasm.yml b/.github/workflows/build-wasm.ymlindex aa7ae887dc..2e4680f385 100644--- a/.github/workflows/build-wasm.yml+++ b/.github/workflows/build-wasm.yml@@ -88,3 +88,13 @@ jobs:-DEMDAWNWEBGPU_DIR=emdawnwebgpu_pkgtime cmake --build build-wasm --config Release --target test-backend-ops -j $(nproc)++ - name: ccache-clear+ uses: ./.github/actions/ccache-clear+ env:+ GH_TOKEN: ${{ github.token }}+ with:+ key: webgpu-ubuntu-24.04-arm-wasm+ older: 5m+ min: 1+ dry-run: ${{ github.event_name != 'push' || github.ref != 'refs/heads/master' }}diff --git a/.github/workflows/build-webgpu.yml b/.github/workflows/build-webgpu.ymlindex ed73c185aa..b357851aa2 100644--- a/.github/workflows/build-webgpu.yml+++ b/.github/workflows/build-webgpu.yml@@ -101,6 +101,16 @@ jobs:cd buildctest -L main --verbose --timeout 900+ - name: ccache-clear+ uses: ./.github/actions/ccache-clear+ env:+ GH_TOKEN: ${{ github.token }}+ with:+ key: webgpu-macos-latest+ older: 5m+ min: 1+ dry-run: ${{ github.event_name != 'push' || github.ref != 'refs/heads/master' }}+ubuntu:runs-on: ubuntu-24.04@@ -153,3 +163,13 @@ jobs:# This is using llvmpipe and runs slower than other backends# test-backend-ops is too slow on llvmpipe, skip itctest -L main -E test-backend-ops --verbose --timeout 900++ - name: ccache-clear+ uses: ./.github/actions/ccache-clear+ env:+ GH_TOKEN: ${{ github.token }}+ with:+ key: webgpu-ubuntu-24.04+ older: 5m+ min: 1+ dry-run: ${{ github.event_name != 'push' || github.ref != 'refs/heads/master' }}diff --git a/.github/workflows/hip-quality-check.yml b/.github/workflows/hip-quality-check.ymlindex 5d23f01cf8..ecc4615a1a 100644--- a/.github/workflows/hip-quality-check.yml+++ b/.github/workflows/hip-quality-check.yml@@ -84,3 +84,13 @@ jobs:cd buildmake -j $(nproc) 2>&1 | tee metrics.log | grep -v 'Rpass-analysis=kernel-resource-usage\|remark:\|^$'python3 ../scripts/hip/gcn-cdna-vgpr-check.py metrics.log++ - name: ccache-clear+ uses: ./.github/actions/ccache-clear+ env:+ GH_TOKEN: ${{ github.token }}+ with:+ key: hip-quality-check-ubuntu-22.04+ older: 5m+ min: 1+ dry-run: ${{ github.event_name != 'push' || github.ref != 'refs/heads/master' }}diff --git a/.github/workflows/server.yml b/.github/workflows/server.ymlindex 9fb4b4ba10..530ace7cda 100644--- a/.github/workflows/server.yml+++ b/.github/workflows/server.yml@@ -128,6 +128,16 @@ jobs:export LLAMA_ARG_BACKEND_SAMPLING=1SLOW_TESTS=1 ./tests.sh+ - name: ccache-clear+ uses: ./.github/actions/ccache-clear+ env:+ GH_TOKEN: ${{ github.token }}+ with:+ key: server-ubuntu-24.04-arm+ older: 5m+ min: 1+ dry-run: ${{ github.event_name != 'push' || github.ref != 'refs/heads/master' }}+windows:runs-on: windows-2025@@ -181,3 +191,13 @@ jobs:cd tools/server/testsexport SLOW_TESTS="1"./tests.sh++ - name: ccache-clear+ uses: ./.github/actions/ccache-clear+ env:+ GH_TOKEN: ${{ github.token }}+ with:+ key: server-windows-2025-x64+ older: 5m+ min: 1+ dry-run: ${{ github.event_name != 'push' || github.ref != 'refs/heads/master' }}diff --git a/scripts/ccache-clear.sh b/scripts/ccache-clear.shnew file mode 100755index 0000000000..27fda33156--- /dev/null+++ b/scripts/ccache-clear.sh@@ -0,0 +1,105 @@+#!/bin/bash+# Delete GitHub Actions caches matching a key prefix, oldest first.+#+# Usage: ccache-clear.sh --key KEY [--older DURATION] [--min N] [--dry-run]+# --key: cache key prefix to match and delete (without the ccache- prefix)+# --older: only delete caches created more than DURATION ago (e.g. 5m, 1h, 1d);+# by default all matching caches are deleted+# --min: stop deleting if fewer than N caches would remain (default: 0)+# --dry-run: only print the caches that would be deleted, without deleting them+#+# Env (when running in GitHub Actions):+# GH_TOKEN: token for the gh CLI+# GITHUB_REPOSITORY: owner/repo of the caches to manage+set -euo pipefail++KEY=""+OLDER=""+MIN=0+DRY_RUN=false+while [[ $# -gt 0 ]]; do+ case "$1" in+ --key) [[ $# -ge 2 ]] || { echo "Missing value for $1" >&2; exit 1; }; KEY="$2"; shift 2 ;;+ --older) [[ $# -ge 2 ]] || { echo "Missing value for $1" >&2; exit 1; }; OLDER="$2"; shift 2 ;;+ --min) [[ $# -ge 2 ]] || { echo "Missing value for $1" >&2; exit 1; }; MIN="$2"; shift 2 ;;+ --dry-run) DRY_RUN=true; shift ;;+ *) echo "Unknown argument: $1"; exit 1 ;;+ esac+done++command -v gh >/dev/null 2>&1 || { echo "Error: GitHub CLI (gh) is required" >&2; exit 1; }+[[ -n "${GITHUB_REPOSITORY:-}" ]] || { echo "Error: GITHUB_REPOSITORY not set" >&2; exit 1; }+[[ -n "$KEY" ]] || { echo "Error: --key is required" >&2; exit 1; }+[[ "$MIN" =~ ^[0-9]+$ ]] || { echo "Invalid min value: $MIN" >&2; exit 1; }++# Convert a duration (e.g. 90m, 1h, 1d, plain seconds) to seconds+to_seconds() {+ local val="$1"+ [[ "$val" =~ ^[0-9]+$ ]] && { echo "$val"; return 0; }+ local num="${val%?}" unit="${val: -1}" mult+ [[ "$num" =~ ^[0-9]+$ ]] || return 1+ case "$unit" in+ s) mult=1 ;;+ m) mult=60 ;;+ h) mult=3600 ;;+ d) mult=86400 ;;+ *) return 1 ;;+ esac+ echo $((num * mult))+}++# Convert an ISO-8601 UTC timestamp (e.g. 2026-08-23T16:51:23.313693Z) to epoch seconds+to_epoch() {+ local val="$1" out+ # GNU date (e.g. Linux)+ if out=$(date -d "$val" +%s 2>/dev/null) && [[ "$out" =~ ^[0-9]+$ ]]; then+ echo "$out"+ return 0+ fi+ # BSD date (e.g. macOS); fractional seconds are not needed, TZ forces UTC+ out=$(TZ=UTC date -j -f "%Y-%m-%dT%H:%M:%S" "${val:0:19}" +%s 2>/dev/null) || return 1+ [[ "$out" =~ ^[0-9]+$ ]] || return 1+ echo "$out"+}++CACHES=$(gh cache list --repo "$GITHUB_REPOSITORY" --key "ccache-$KEY" --json id,key,createdAt --jq '.[] | [.createdAt, .id, .key] | @tsv' | LC_ALL=C sort)+if [[ -z "$CACHES" ]]; then+ echo "No caches found with key prefix: $KEY"+ exit 0+fi++TOTAL=$(( $(wc -l <<< "$CACHES") ))++echo "Found $TOTAL cache(s) with key prefix: $KEY (oldest first):"+while IFS=$'\t' read -r CREATED ID CACHE_KEY; do+ printf ' %s %s %s\n' "$CREATED" "$ID" "$CACHE_KEY"+done <<< "$CACHES"++CUTOFF=""+if [[ -n "$OLDER" ]]; then+ OLDER_SECONDS=$(to_seconds "$OLDER") || { echo "Invalid older value: $OLDER (expected e.g. 90m, 1h, 1d)" >&2; exit 1; }+ CUTOFF=$(( $(date +%s) - OLDER_SECONDS ))+fi++# Caches are sorted oldest first+DELETED=0+while IFS=$'\t' read -r CREATED ID CACHE_KEY; do+ if [[ -n "$CUTOFF" ]]; then+ CREATED_SECONDS=$(to_epoch "$CREATED") || { echo "Failed to parse date: $CREATED" >&2; exit 1; }+ if [[ "$CREATED_SECONDS" -ge "$CUTOFF" ]]; then+ echo "Rest are not older than $OLDER, stopping"+ break+ fi+ fi+ if (( TOTAL - DELETED - 1 < MIN )); then+ echo "Keeping at least $MIN cache(s), stopping"+ break+ fi+ if [[ "$DRY_RUN" == "true" ]]; then+ echo "Would delete cache: $ID ($CACHE_KEY)"+ else+ echo "Deleting cache: $ID ($CACHE_KEY)"+ gh cache delete --repo "$GITHUB_REPOSITORY" "$ID"+ fi+ DELETED=$((DELETED + 1))+done <<< "$CACHES"