Skip to content

LLM inference in C/C++

C++ ggml

ci : run test-backend-ops as a dedicated ci/run.sh test (#28740)

Georgi Gerganovb78a39a2f93b
diff --git a/.github/workflows/build-openvino.yml b/.github/workflows/build-openvino.yml
index 8879a6af16..86aba456ce 100644
--- a/.github/workflows/build-openvino.yml
+++ b/.github/workflows/build-openvino.yml
@@ -33,7 +33,7 @@ env:
LLAMA_ARG_LOG_PREFIX: 1
LLAMA_ARG_LOG_TIMESTAMPS: 1
# TODO: fix failing tests on OpenVINO backend
- CTEST_EXCLUDE: "test-llama-archs|^test-recurrent-state-|test-backend-ops|test-save-load-state"
+ CTEST_EXCLUDE: "test-llama-archs|^test-recurrent-state-|test-save-load-state"
jobs:
ubuntu-24-openvino:
diff --git a/.github/workflows/build-self-hosted.yml b/.github/workflows/build-self-hosted.yml
index fda4879e21..02a38466fa 100644
--- a/.github/workflows/build-self-hosted.yml
+++ b/.github/workflows/build-self-hosted.yml
@@ -395,7 +395,11 @@ jobs:
- name: Test
id: ggml-ci
run: |
- LLAMA_ARG_THREADS=$(nproc) GG_BUILD_HIGH_PERF=1 GG_BUILD_NO_BF16=1 GG_BUILD_EXTRA_TESTS_0=1 bash ./ci/run.sh ~/results/llama.cpp ~/mnt/llama.cpp
+ LLAMA_ARG_THREADS=$(nproc) \
+ GG_BUILD_HIGH_PERF=1 \
+ GG_BUILD_NO_BF16=1 \
+ GG_BUILD_EXTRA_TESTS_0=1 \
+ bash ./ci/run.sh ~/results/llama.cpp ~/mnt/llama.cpp
cpu-arm64-graviton4-kleidiai:
runs-on: ah-ubuntu_22_04-c8g_8x
@@ -434,6 +438,8 @@ jobs:
- name: Test
id: ggml-ci
run: |
+ LLAMA_ARG_THREADS=$(nproc) \
GG_BUILD_KLEIDIAI=1 \
GG_BUILD_EXTRA_TESTS_0=1 \
- bash ./ci/run.sh ./tmp/results ./tmp/mnt
+ GG_BUILD_HIGH_PERF=1 \
+ bash ./ci/run.sh ~/results/llama.cpp ~/mnt/llama.cpp
diff --git a/.github/workflows/build-vulkan.yml b/.github/workflows/build-vulkan.yml
index 9de52e990a..21d2a77353 100644
--- a/.github/workflows/build-vulkan.yml
+++ b/.github/workflows/build-vulkan.yml
@@ -164,9 +164,7 @@ jobs:
export GGML_VK_VISIBLE_DEVICES=0
export GGML_VK_DISABLE_F16=1
export GGML_VK_DISABLE_COOPMAT=1
- # This is using llvmpipe and runs slower than other backends
- # test-backend-ops is too slow on llvmpipe, skip it
- ctest -L main -E test-backend-ops --verbose --timeout 900
+ ctest -L main --verbose --timeout 900
windows:
runs-on: windows-2025
diff --git a/.github/workflows/build-webgpu.yml b/.github/workflows/build-webgpu.yml
index e624e3ba80..8277abcc47 100644
--- a/.github/workflows/build-webgpu.yml
+++ b/.github/workflows/build-webgpu.yml
@@ -190,6 +190,4 @@ jobs:
id: cmake_test
run: |
cd build
- # This is using llvmpipe and runs slower than other backends
- # test-backend-ops is too slow on llvmpipe, skip it
- ctest -L main -E test-backend-ops --verbose --timeout 900
+ ctest -L main --verbose --timeout 900
diff --git a/ci/run.sh b/ci/run.sh
index 294cbe57bb..1ceb19fd50 100755
--- a/ci/run.sh
+++ b/ci/run.sh
@@ -190,7 +190,7 @@ if [ ! -z ${GG_BUILD_OPENVINO} ]; then
CMAKE_EXTRA="${CMAKE_EXTRA} -DGGML_OPENVINO=ON"
# TODO: fix failing tests on OpenVINO backend
- CTEST_EXTRA="-E test-llama-archs|^test-recurrent-state-|test-backend-ops|test-save-load-state"
+ CTEST_EXTRA="-E test-llama-archs|^test-recurrent-state-|test-save-load-state"
fi
## helpers
@@ -250,7 +250,7 @@ function gg_run_ctest_debug {
(cmake -G "${CMAKE_GENERATOR}" -DCMAKE_BUILD_TYPE=Debug ${CMAKE_EXTRA} .. ) 2>&1 | tee -a $OUT/${ci}-cmake.log
(time cmake --build . --config Debug -j$(nproc)) 2>&1 | tee -a $OUT/${ci}-make.log
- (time ctest -C Debug --output-on-failure -L main -E "test-opt|test-backend-ops|test-llama-archs" ${CTEST_EXTRA}) 2>&1 | tee -a $OUT/${ci}-ctest.log
+ (time ctest -C Debug --output-on-failure -L main -E "test-opt|test-llama-archs" ${CTEST_EXTRA}) 2>&1 | tee -a $OUT/${ci}-ctest.log
set +e
}
@@ -768,25 +768,43 @@ function gg_check_build_requirements {
fi
}
-function gg_run_test_backend_ops_cpu {
+function gg_run_test_backend_ops {
cd ${SRC}
cd build-ci-release
set -e
- (time ./bin/test-backend-ops -b CPU ) 2>&1 | tee -a $OUT/${ci}-test-backend-ops-cpu.log
+ local args_extra="-j $(nproc)"
+
+ # TODO: fix multi-threaded for ROCm
+ # https://github.com/ggml-org/llama.cpp/actions/runs/34576278519/job/103297889044?pr=28740#step:3:4865
+ if [ ! -z ${GG_BUILD_ROCM} ]; then
+ args_extra=""
+ fi
+
+ # TODO: MoltenVK bug?
+ # https://github.com/ggml-org/llama.cpp/actions/runs/34611260059/job/103302413736?pr=28740#step:3:5897
+ if [ ! -z "${GG_BUILD_VULKAN}" ] && [ "$(uname -s)" = "Darwin" ]; then
+ args_extra=""
+ fi
+
+ if [ ! -z ${GG_BUILD_HIGH_PERF} ]; then
+ (time ./bin/test-backend-ops ${args_extra} -b CPU) 2>&1 | tee -a $OUT/${ci}-test-backend-ops.log
+ else
+ (time ./bin/test-backend-ops ${args_extra} ) 2>&1 | tee -a $OUT/${ci}-test-backend-ops.log
+ fi
set +e
}
-function gg_sum_test_backend_ops_cpu {
+function gg_sum_test_backend_ops {
gg_printf '### %s\n\n' "${ci}"
- gg_printf 'Runs test-backend-ops for CPU backend\n'
+ gg_printf 'Runs test-backend-ops\n'
gg_printf '- status: %s\n' "$(cat $OUT/${ci}.exit)"
gg_printf '```\n'
- gg_printf '%s\n' "$(cat $OUT/${ci}-test-backend-ops-cpu.log)"
+ gg_printf '%s\n' "$(cat $OUT/${ci}-test-backend-ops.log)"
gg_printf '```\n'
gg_printf '\n'
}
@@ -819,13 +837,11 @@ ret=0
test $ret -eq 0 && gg_run ctest_debug
test $ret -eq 0 && gg_run ctest_release
+test $ret -eq 0 && gg_run test_backend_ops
+
test $ret -eq 0 && gg_run test_llama_archs_models
test $ret -eq 0 && gg_run test_llama_archs_tensor_split
-if [ ! -z ${GG_BUILD_HIGH_PERF} ]; then
- test $ret -eq 0 && gg_run test_backend_ops_cpu
-fi
-
if [ -z ${GG_BUILD_LOW_PERF} ]; then
test $ret -eq 0 && gg_run embd_bge_small
test $ret -eq 0 && gg_run rerank_tiny
diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt
index 0c4e4d5a9b..b3559a173f 100644
--- a/tests/CMakeLists.txt
+++ b/tests/CMakeLists.txt
@@ -10,7 +10,7 @@ function(llama_build source)
endif()
add_executable(${TEST_TARGET} ${TEST_SOURCES})
- target_link_libraries(${TEST_TARGET} PRIVATE llama llama-common)
+ target_link_libraries(${TEST_TARGET} PRIVATE llama llama-common) # TODO: [TAG_TESTS_LLAMA_LINK]
if (LLAMA_TESTS_INSTALL)
install(TARGETS ${TEST_TARGET} RUNTIME)
endif()
@@ -310,15 +310,9 @@ if (NOT LLAMA_SANITIZE_ADDRESS AND NOT GGML_SCHED_NO_REALLOC)
# TODO: repair known memory leaks
llama_build_and_test(test-opt.cpp)
endif()
-llama_build_and_test(test-backend-ops.cpp)
-
-# the tensor API kernels come from a separate metallib - check they produce correct results
-# ref: https://github.com/ggml-org/llama.cpp/issues/27473
-if (GGML_METAL AND NOT GGML_METAL_EMBED_LIBRARY)
- llama_test(test-backend-ops NAME test-backend-ops-metallib-tensor
- ARGS test -b MTL0 -o MUL_MAT -p type_a=q6_K)
- set_tests_properties(test-backend-ops-metallib-tensor PROPERTIES ENVIRONMENT GGML_METAL_TENSOR_ENABLE=1)
-endif()
+
+# TODO: make this test (and others) not link `libllama` as it is not needed [TAG_TESTS_LLAMA_LINK]
+llama_build(test-backend-ops.cpp)
llama_build_and_test(test-model-load-cancel.cpp LABEL "model")
llama_build_and_test(test-autorelease.cpp LABEL "model")