Skip to content

LLM inference in C/C++

C++ ggml

tests : clean-up server test, use `tests.sh` in ci (#26886)

Georgi Gerganovb3df57286c33
diff --git a/.github/workflows/server-sanitize.yml b/.github/workflows/server-sanitize.yml
index 0eeefdf884..5d696282c7 100644
--- a/.github/workflows/server-sanitize.yml
+++ b/.github/workflows/server-sanitize.yml
@@ -110,7 +110,7 @@ jobs:
source .venv/bin/activate
cd tools/server/tests
export ${{ matrix.extra_args }}
- pytest -v -x -m "not slow"
+ ./tests.sh
- name: Slow tests
id: server_integration_tests_slow
@@ -119,4 +119,4 @@ jobs:
source .venv/bin/activate
cd tools/server/tests
export ${{ matrix.extra_args }}
- SLOW_TESTS=1 pytest -v -x
+ SLOW_TESTS=1 ./tests.sh
diff --git a/.github/workflows/server-self-hosted.yml b/.github/workflows/server-self-hosted.yml
index 249f389ff3..675ddbaaa5 100644
--- a/.github/workflows/server-self-hosted.yml
+++ b/.github/workflows/server-self-hosted.yml
@@ -72,7 +72,7 @@ jobs:
run: |
cd tools/server/tests
source venv/bin/activate
- pytest -v -x -m "not slow"
+ ./tests.sh
- name: Tests (GPUx1, backend-sampling)
id: server_integration_tests_backend_sampling
@@ -81,7 +81,7 @@ jobs:
cd tools/server/tests
source venv/bin/activate
export LLAMA_ARG_BACKEND_SAMPLING=1
- pytest -v -x -m "not slow"
+ ./tests.sh
- name: Tests (GPUx2)
id: server_integration_tests_gpu2
@@ -90,7 +90,7 @@ jobs:
cd tools/server/tests
source venv/bin/activate
export GGML_METAL_DEVICES=2
- pytest -v -x -m "not slow"
+ ./tests.sh
- name: Tests (GPUx2, backend-sampling)
id: server_integration_tests_gpu2_backend_sampling
@@ -99,7 +99,7 @@ jobs:
cd tools/server/tests
source venv/bin/activate
export GGML_METAL_DEVICES=2 LLAMA_ARG_BACKEND_SAMPLING=1
- pytest -v -x -m "not slow"
+ ./tests.sh
server-cuda:
runs-on: [self-hosted, llama-server, Linux, NVIDIA]
@@ -132,7 +132,7 @@ jobs:
run: |
cd tools/server/tests
source venv/bin/activate
- pytest -v -x -m "not slow"
+ ./tests.sh
- name: Tests (GPUx1, backend-sampling)
id: server_integration_tests_backend_sampling
@@ -141,7 +141,7 @@ jobs:
cd tools/server/tests
source venv/bin/activate
export LLAMA_ARG_BACKEND_SAMPLING=1
- pytest -v -x -m "not slow"
+ ./tests.sh
- name: Tests (GPUx2)
id: server_integration_tests_gpu2
@@ -150,7 +150,7 @@ jobs:
cd tools/server/tests
source venv/bin/activate
export GGML_CUDA_DEVICES=2
- pytest -v -x -m "not slow"
+ ./tests.sh
- name: Tests (GPUx2, backend-sampling)
id: server_integration_tests_gpu2_backend_sampling
@@ -159,7 +159,7 @@ jobs:
cd tools/server/tests
source venv/bin/activate
export GGML_CUDA_DEVICES=2 LLAMA_ARG_BACKEND_SAMPLING=1
- pytest -v -x -m "not slow"
+ ./tests.sh
server-kleidiai:
runs-on: ah-ubuntu_22_04-c8g_8x
@@ -219,4 +219,4 @@ jobs:
run: |
cd tools/server/tests
source venv/bin/activate
- pytest -v -x -m "not slow"
+ ./tests.sh
diff --git a/.github/workflows/server.yml b/.github/workflows/server.yml
index 5a02cc15ad..d5abf1d236 100644
--- a/.github/workflows/server.yml
+++ b/.github/workflows/server.yml
@@ -104,21 +104,21 @@ jobs:
id: server_integration_tests
run: |
cd tools/server/tests
- pytest -v -x -m "not slow"
+ ./tests.sh
- name: Slow tests
id: server_integration_tests_slow
if: ${{ github.event.schedule || github.event.inputs.slow_tests == 'true' }}
run: |
cd tools/server/tests
- SLOW_TESTS=1 pytest -v -x
+ SLOW_TESTS=1 ./tests.sh
- name: Tests (Backend sampling)
id: server_integration_tests_backend_sampling
run: |
cd tools/server/tests
export LLAMA_ARG_BACKEND_SAMPLING=1
- pytest -v -x -m "not slow"
+ ./tests.sh
- name: Slow tests (Backend sampling)
id: server_integration_tests_slow_backend_sampling
@@ -126,7 +126,7 @@ jobs:
run: |
cd tools/server/tests
export LLAMA_ARG_BACKEND_SAMPLING=1
- SLOW_TESTS=1 pytest -v -x
+ SLOW_TESTS=1 ./tests.sh
windows:
runs-on: windows-2025
@@ -170,7 +170,7 @@ jobs:
run: |
cd tools/server/tests
$env:PYTHONIOENCODING = ":replace"
- pytest -v -x -m "not slow"
+ ./tests.sh
- name: Slow tests
id: server_integration_tests_slow
@@ -178,4 +178,4 @@ jobs:
run: |
cd tools/server/tests
$env:SLOW_TESTS = "1"
- pytest -v -x
+ ./tests.sh
diff --git a/scripts/fetch_server_test_models.py b/scripts/fetch_server_test_models.py
deleted file mode 100755
index f43d1f63cd..0000000000
--- a/scripts/fetch_server_test_models.py
+++ /dev/null
@@ -1,105 +0,0 @@
-#!/usr/bin/env python
-'''
- This script fetches all the models used in the server tests.
-
- This is useful for slow tests that use larger models, to avoid them timing out on the model downloads.
-
- It is meant to be run from the root of the repository.
-
- Example:
- python scripts/fetch_server_test_models.py
- ( cd tools/server/tests && ./tests.sh -v -x -m slow )
-'''
-import ast
-import glob
-import logging
-import os
-from typing import Generator
-from pydantic import BaseModel
-from typing import Optional
-import subprocess
-
-
-class HuggingFaceModel(BaseModel):
- hf_repo: str
- hf_file: Optional[str] = None
-
- class Config:
- frozen = True
-
-
-def collect_hf_model_test_parameters(test_file) -> Generator[HuggingFaceModel, None, None]:
- try:
- with open(test_file) as f:
- tree = ast.parse(f.read())
- except Exception as e:
- logging.error(f'collect_hf_model_test_parameters failed on {test_file}: {e}')
- return
-
- for node in ast.walk(tree):
- if isinstance(node, ast.FunctionDef):
- for dec in node.decorator_list:
- if isinstance(dec, ast.Call) and isinstance(dec.func, ast.Attribute) and dec.func.attr == 'parametrize':
- param_names = ast.literal_eval(dec.args[0]).split(",")
- if "hf_repo" not in param_names:
- continue
-
- raw_param_values = dec.args[1]
- if not isinstance(raw_param_values, ast.List):
- logging.warning(f'Skipping non-list parametrize entry at {test_file}:{node.lineno}')
- continue
-
- hf_repo_idx = param_names.index("hf_repo")
- hf_file_idx = param_names.index("hf_file") if "hf_file" in param_names else None
-
- for t in raw_param_values.elts:
- if not isinstance(t, ast.Tuple):
- logging.warning(f'Skipping non-tuple parametrize entry at {test_file}:{node.lineno}')
- continue
- yield HuggingFaceModel(
- hf_repo=ast.literal_eval(t.elts[hf_repo_idx]),
- hf_file=ast.literal_eval(t.elts[hf_file_idx]) if hf_file_idx is not None else None)
-
-
-if __name__ == '__main__':
- logging.basicConfig(level=logging.INFO, format='%(levelname)s: %(message)s')
-
- models = sorted(list(set([
- model
- for test_file in glob.glob('tools/server/tests/unit/test_*.py')
- for model in collect_hf_model_test_parameters(test_file)
- ])), key=lambda m: (m.hf_repo, m.hf_file))
-
- logging.info(f'Found {len(models)} models in parameterized tests:')
- for m in models:
- logging.info(f' - {m.hf_repo} / {m.hf_file}')
-
- cli_path = os.environ.get(
- 'LLAMA_CLI_BIN_PATH',
- os.path.join(
- os.path.dirname(__file__),
- '../build/bin/Release/llama-cli.exe' if os.name == 'nt' else '../build/bin/llama-cli'))
-
- for m in models:
- if '<' in m.hf_repo or (m.hf_file is not None and '<' in m.hf_file):
- continue
- if m.hf_file is not None and '-of-' in m.hf_file:
- logging.warning(f'Skipping model at {m.hf_repo} / {m.hf_file} because it is a split file')
- continue
- logging.info(f'Using llama-cli to ensure model {m.hf_repo}/{m.hf_file} was fetched')
- cmd = [
- cli_path,
- '-hfr', m.hf_repo,
- *([] if m.hf_file is None else ['-hff', m.hf_file]),
- '-n', '1',
- '-p', 'Hey',
- '--no-warmup',
- '--log-disable',
- '-st']
- if m.hf_file != 'tinyllamas/stories260K.gguf' and 'Mistral-Nemo' not in m.hf_repo:
- cmd += ('-fa', 'on')
- try:
- subprocess.check_call(cmd)
- except subprocess.CalledProcessError:
- logging.error(f'Failed to fetch model at {m.hf_repo} / {m.hf_file} with command:\n {" ".join(cmd)}')
- exit(1)
diff --git a/tools/server/tests/tests.sh b/tools/server/tests/tests.sh
index 8d6681193d..433dc99828 100755
--- a/tools/server/tests/tests.sh
+++ b/tools/server/tests/tests.sh
@@ -6,11 +6,6 @@ cd $SCRIPT_DIR
set -eu
-if [[ "${SLOW_TESTS:-0}" == 1 ]]; then
- # Slow tests for tool calls need quite a few models ahead of time to avoid timing out.
- python $SCRIPT_DIR/../../../scripts/fetch_server_test_models.py
-fi
-
if [ $# -lt 1 ]
then
if [[ "${SLOW_TESTS:-0}" == 1 ]]; then