LLM inference in C/C++
C++ ggml
tests : clean-up server test, use `tests.sh` in ci (#26886)
Georgi Gerganovb3df57286c33
diff --git a/.github/workflows/server-sanitize.yml b/.github/workflows/server-sanitize.ymlindex 0eeefdf884..5d696282c7 100644--- a/.github/workflows/server-sanitize.yml+++ b/.github/workflows/server-sanitize.yml@@ -110,7 +110,7 @@ jobs:source .venv/bin/activatecd tools/server/testsexport ${{ matrix.extra_args }}- pytest -v -x -m "not slow"+ ./tests.sh- name: Slow testsid: server_integration_tests_slow@@ -119,4 +119,4 @@ jobs:source .venv/bin/activatecd tools/server/testsexport ${{ matrix.extra_args }}- SLOW_TESTS=1 pytest -v -x+ SLOW_TESTS=1 ./tests.shdiff --git a/.github/workflows/server-self-hosted.yml b/.github/workflows/server-self-hosted.ymlindex 249f389ff3..675ddbaaa5 100644--- a/.github/workflows/server-self-hosted.yml+++ b/.github/workflows/server-self-hosted.yml@@ -72,7 +72,7 @@ jobs:run: |cd tools/server/testssource venv/bin/activate- pytest -v -x -m "not slow"+ ./tests.sh- name: Tests (GPUx1, backend-sampling)id: server_integration_tests_backend_sampling@@ -81,7 +81,7 @@ jobs:cd tools/server/testssource venv/bin/activateexport LLAMA_ARG_BACKEND_SAMPLING=1- pytest -v -x -m "not slow"+ ./tests.sh- name: Tests (GPUx2)id: server_integration_tests_gpu2@@ -90,7 +90,7 @@ jobs:cd tools/server/testssource venv/bin/activateexport GGML_METAL_DEVICES=2- pytest -v -x -m "not slow"+ ./tests.sh- name: Tests (GPUx2, backend-sampling)id: server_integration_tests_gpu2_backend_sampling@@ -99,7 +99,7 @@ jobs:cd tools/server/testssource venv/bin/activateexport GGML_METAL_DEVICES=2 LLAMA_ARG_BACKEND_SAMPLING=1- pytest -v -x -m "not slow"+ ./tests.shserver-cuda:runs-on: [self-hosted, llama-server, Linux, NVIDIA]@@ -132,7 +132,7 @@ jobs:run: |cd tools/server/testssource venv/bin/activate- pytest -v -x -m "not slow"+ ./tests.sh- name: Tests (GPUx1, backend-sampling)id: server_integration_tests_backend_sampling@@ -141,7 +141,7 @@ jobs:cd tools/server/testssource venv/bin/activateexport LLAMA_ARG_BACKEND_SAMPLING=1- pytest -v -x -m "not slow"+ ./tests.sh- name: Tests (GPUx2)id: server_integration_tests_gpu2@@ -150,7 +150,7 @@ jobs:cd tools/server/testssource venv/bin/activateexport GGML_CUDA_DEVICES=2- pytest -v -x -m "not slow"+ ./tests.sh- name: Tests (GPUx2, backend-sampling)id: server_integration_tests_gpu2_backend_sampling@@ -159,7 +159,7 @@ jobs:cd tools/server/testssource venv/bin/activateexport GGML_CUDA_DEVICES=2 LLAMA_ARG_BACKEND_SAMPLING=1- pytest -v -x -m "not slow"+ ./tests.shserver-kleidiai:runs-on: ah-ubuntu_22_04-c8g_8x@@ -219,4 +219,4 @@ jobs:run: |cd tools/server/testssource venv/bin/activate- pytest -v -x -m "not slow"+ ./tests.shdiff --git a/.github/workflows/server.yml b/.github/workflows/server.ymlindex 5a02cc15ad..d5abf1d236 100644--- a/.github/workflows/server.yml+++ b/.github/workflows/server.yml@@ -104,21 +104,21 @@ jobs:id: server_integration_testsrun: |cd tools/server/tests- pytest -v -x -m "not slow"+ ./tests.sh- name: Slow testsid: server_integration_tests_slowif: ${{ github.event.schedule || github.event.inputs.slow_tests == 'true' }}run: |cd tools/server/tests- SLOW_TESTS=1 pytest -v -x+ SLOW_TESTS=1 ./tests.sh- name: Tests (Backend sampling)id: server_integration_tests_backend_samplingrun: |cd tools/server/testsexport LLAMA_ARG_BACKEND_SAMPLING=1- pytest -v -x -m "not slow"+ ./tests.sh- name: Slow tests (Backend sampling)id: server_integration_tests_slow_backend_sampling@@ -126,7 +126,7 @@ jobs:run: |cd tools/server/testsexport LLAMA_ARG_BACKEND_SAMPLING=1- SLOW_TESTS=1 pytest -v -x+ SLOW_TESTS=1 ./tests.shwindows:runs-on: windows-2025@@ -170,7 +170,7 @@ jobs:run: |cd tools/server/tests$env:PYTHONIOENCODING = ":replace"- pytest -v -x -m "not slow"+ ./tests.sh- name: Slow testsid: server_integration_tests_slow@@ -178,4 +178,4 @@ jobs:run: |cd tools/server/tests$env:SLOW_TESTS = "1"- pytest -v -x+ ./tests.shdiff --git a/scripts/fetch_server_test_models.py b/scripts/fetch_server_test_models.pydeleted file mode 100755index f43d1f63cd..0000000000--- a/scripts/fetch_server_test_models.py+++ /dev/null@@ -1,105 +0,0 @@-#!/usr/bin/env python-'''- This script fetches all the models used in the server tests.-- This is useful for slow tests that use larger models, to avoid them timing out on the model downloads.-- It is meant to be run from the root of the repository.-- Example:- python scripts/fetch_server_test_models.py- ( cd tools/server/tests && ./tests.sh -v -x -m slow )-'''-import ast-import glob-import logging-import os-from typing import Generator-from pydantic import BaseModel-from typing import Optional-import subprocess---class HuggingFaceModel(BaseModel):- hf_repo: str- hf_file: Optional[str] = None-- class Config:- frozen = True---def collect_hf_model_test_parameters(test_file) -> Generator[HuggingFaceModel, None, None]:- try:- with open(test_file) as f:- tree = ast.parse(f.read())- except Exception as e:- logging.error(f'collect_hf_model_test_parameters failed on {test_file}: {e}')- return-- for node in ast.walk(tree):- if isinstance(node, ast.FunctionDef):- for dec in node.decorator_list:- if isinstance(dec, ast.Call) and isinstance(dec.func, ast.Attribute) and dec.func.attr == 'parametrize':- param_names = ast.literal_eval(dec.args[0]).split(",")- if "hf_repo" not in param_names:- continue-- raw_param_values = dec.args[1]- if not isinstance(raw_param_values, ast.List):- logging.warning(f'Skipping non-list parametrize entry at {test_file}:{node.lineno}')- continue-- hf_repo_idx = param_names.index("hf_repo")- hf_file_idx = param_names.index("hf_file") if "hf_file" in param_names else None-- for t in raw_param_values.elts:- if not isinstance(t, ast.Tuple):- logging.warning(f'Skipping non-tuple parametrize entry at {test_file}:{node.lineno}')- continue- yield HuggingFaceModel(- hf_repo=ast.literal_eval(t.elts[hf_repo_idx]),- hf_file=ast.literal_eval(t.elts[hf_file_idx]) if hf_file_idx is not None else None)---if __name__ == '__main__':- logging.basicConfig(level=logging.INFO, format='%(levelname)s: %(message)s')-- models = sorted(list(set([- model- for test_file in glob.glob('tools/server/tests/unit/test_*.py')- for model in collect_hf_model_test_parameters(test_file)- ])), key=lambda m: (m.hf_repo, m.hf_file))-- logging.info(f'Found {len(models)} models in parameterized tests:')- for m in models:- logging.info(f' - {m.hf_repo} / {m.hf_file}')-- cli_path = os.environ.get(- 'LLAMA_CLI_BIN_PATH',- os.path.join(- os.path.dirname(__file__),- '../build/bin/Release/llama-cli.exe' if os.name == 'nt' else '../build/bin/llama-cli'))-- for m in models:- if '<' in m.hf_repo or (m.hf_file is not None and '<' in m.hf_file):- continue- if m.hf_file is not None and '-of-' in m.hf_file:- logging.warning(f'Skipping model at {m.hf_repo} / {m.hf_file} because it is a split file')- continue- logging.info(f'Using llama-cli to ensure model {m.hf_repo}/{m.hf_file} was fetched')- cmd = [- cli_path,- '-hfr', m.hf_repo,- *([] if m.hf_file is None else ['-hff', m.hf_file]),- '-n', '1',- '-p', 'Hey',- '--no-warmup',- '--log-disable',- '-st']- if m.hf_file != 'tinyllamas/stories260K.gguf' and 'Mistral-Nemo' not in m.hf_repo:- cmd += ('-fa', 'on')- try:- subprocess.check_call(cmd)- except subprocess.CalledProcessError:- logging.error(f'Failed to fetch model at {m.hf_repo} / {m.hf_file} with command:\n {" ".join(cmd)}')- exit(1)diff --git a/tools/server/tests/tests.sh b/tools/server/tests/tests.shindex 8d6681193d..433dc99828 100755--- a/tools/server/tests/tests.sh+++ b/tools/server/tests/tests.sh@@ -6,11 +6,6 @@ cd $SCRIPT_DIRset -eu-if [[ "${SLOW_TESTS:-0}" == 1 ]]; then- # Slow tests for tool calls need quite a few models ahead of time to avoid timing out.- python $SCRIPT_DIR/../../../scripts/fetch_server_test_models.py-fi-if [ $# -lt 1 ]thenif [[ "${SLOW_TESTS:-0}" == 1 ]]; then