LLM inference in C/C++
C++ ggml
server : use pytest-xdist for server tests (#28298)
Daniel Bevenius42f0225fea94
diff --git a/.github/workflows/server-sanitize.yml b/.github/workflows/server-sanitize.ymlindex a0ffc70c12..77549ee871 100644--- a/.github/workflows/server-sanitize.yml+++ b/.github/workflows/server-sanitize.yml@@ -103,7 +103,7 @@ jobs:source .venv/bin/activatecd tools/server/testsexport ${{ matrix.extra_args }}- ./tests.sh+ PYTEST_WORKERS=1 ./tests.sh- name: Slow testsid: server_integration_tests_slow@@ -112,4 +112,4 @@ jobs:source .venv/bin/activatecd tools/server/testsexport ${{ matrix.extra_args }}- SLOW_TESTS=1 ./tests.sh+ PYTEST_WORKERS=1 SLOW_TESTS=1 ./tests.shdiff --git a/tools/server/tests/conftest.py b/tools/server/tests/conftest.pyindex 5dfde40796..69c4fd687f 100644--- a/tools/server/tests/conftest.py+++ b/tools/server/tests/conftest.py@@ -1,7 +1,17 @@+import osimport pytest+from filelock import FileLockfrom utils import *[email protected](scope="session", autouse=True)+def configure_worker_port(request):+ worker_id = getattr(request.config, "workerinput", {}).get("workerid", "master")+ if worker_id != "master":+ worker_num = int(worker_id[2:])+ os.environ["PORT"] = str(8080 + worker_num * 10)++# ref: https://stackoverflow.com/questions/22627659/run-code-before-and-after-each-test-in-py-test@pytest.fixture(autouse=True)def stop_server_after_each_test():@@ -16,6 +26,10 @@ def stop_server_after_each_test():@pytest.fixture(scope="session", autouse=True)-def load_server_presets():+def load_server_presets(configure_worker_port, tmp_path_factory):# this will be run once per test session, before any tests- ServerPreset.load_all()++ # serialize model downloads across parallel workers.+ root_tmp_dir = tmp_path_factory.getbasetemp().parent+ with FileLock(str(root_tmp_dir / "load_all.lock")):+ ServerPreset.load_all()diff --git a/tools/server/tests/requirements.txt b/tools/server/tests/requirements.txtindex ca7a0281fa..6c256f67d8 100644--- a/tools/server/tests/requirements.txt+++ b/tools/server/tests/requirements.txt@@ -1,5 +1,7 @@aiohttp~=3.9.3pytest~=8.3.3+pytest-xdist~=3.6+filelock~=3.16numpy~=1.26.4openai~=2.14.0prometheus-client~=0.20.0diff --git a/tools/server/tests/tests.sh b/tools/server/tests/tests.shindex 433dc99828..1d1415663b 100755--- a/tools/server/tests/tests.sh+++ b/tools/server/tests/tests.sh@@ -6,13 +6,15 @@ cd $SCRIPT_DIRset -eu+WORKERS="${PYTEST_WORKERS:-auto}"+if [ $# -lt 1 ]thenif [[ "${SLOW_TESTS:-0}" == 1 ]]; then- pytest --durations=30 -v -x+ pytest --durations=30 -v -x -n "${WORKERS}" --dist=workstealelse- pytest --durations=30 -v -x -m "not slow"+ pytest --durations=30 -v -x -n "${WORKERS}" --dist=worksteal -m "not slow"fielse- pytest --durations=30 "$@"+ pytest --durations=30 -n "${WORKERS}" --dist=worksteal "$@"fidiff --git a/tools/server/tests/unit/test_compat_anthropic.py b/tools/server/tests/unit/test_compat_anthropic.pyindex e23947cdde..d292f83cf0 100644--- a/tools/server/tests/unit/test_compat_anthropic.py+++ b/tools/server/tests/unit/test_compat_anthropic.py@@ -21,7 +21,6 @@ def create_server():global serverserver = ServerPreset.tinyllama2()server.model_alias = "tinyllama-2-anthropic"- server.server_port = 8082server.n_slots = 1server.n_ctx = 8192server.n_batch = 2048@@ -34,7 +33,6 @@ def vision_server():server = ServerPreset.tinygemma3()server.offline = False # Allow downloading the modelserver.model_alias = "tinygemma3-anthropic"- server.server_port = 8083 # Different port to avoid conflictsserver.n_slots = 1return server@@ -1015,7 +1013,6 @@ def test_anthropic_thinking_with_reasoning_model(stream):server.jinja = Trueserver.n_ctx = 8192server.n_predict = 1024- server.server_port = 8084server.start(timeout_seconds=600) # large model needs time to downloadif stream:diff --git a/tools/server/tests/unit/test_mcp_servers.py b/tools/server/tests/unit/test_mcp_servers.pyindex 9ad2241bd0..877b732a53 100644--- a/tools/server/tests/unit/test_mcp_servers.py+++ b/tools/server/tests/unit/test_mcp_servers.py@@ -37,7 +37,6 @@ def _start_server_with_mcp(mcp_json: str, **kwargs) -> ServerProcess:srv = ServerPreset.router()srv.server_tools = "all"srv.no_ui = True- srv.server_port = 8085 # avoid conflict with load_all() which uses 8080srv.mcp_servers_json = mcp_jsonfor k, v in kwargs.items():setattr(srv, k, v)@@ -183,7 +182,6 @@ def test_mcp_tools_not_listed_when_not_configured():server = ServerPreset.router()server.server_tools = "all"server.no_ui = True- server.server_port = 8085server.start()try:@@ -250,7 +248,6 @@ def test_mcp_tools_via_json_config_file():server = ServerPreset.router()server.server_tools = "all"server.no_ui = True- server.server_port = 8085server.mcp_servers_config = config_pathserver.start()@@ -468,7 +465,6 @@ def test_mcp_config_file_errors():server = ServerPreset.router()server.server_tools = "all"server.no_ui = True- server.server_port = 8085server.mcp_servers_json = "not valid json"try:server.start()@@ -480,7 +476,6 @@ def test_mcp_config_file_errors():server = ServerPreset.router()server.server_tools = "all"server.no_ui = True- server.server_port = 8085server.mcp_servers_config = "/nonexistent/path.json"try:server.start()diff --git a/tools/server/tests/unit/test_slot_save.py b/tools/server/tests/unit/test_slot_save.pyindex 5af61d70d0..5eca46cb29 100644--- a/tools/server/tests/unit/test_slot_save.py+++ b/tools/server/tests/unit/test_slot_save.py@@ -10,10 +10,10 @@ STATE_FILE_HEADER_SIZE = 12server = ServerPreset.tinyllama2()@pytest.fixture(autouse=True)-def create_server():+def create_server(tmp_path):global serverserver = ServerPreset.tinyllama2()- server.slot_save_path = "./tmp"+ server.slot_save_path = str(tmp_path)server.temperature = 0.0@@ -94,7 +94,7 @@ def test_slot_restore_legacy_token_list():assert res.body["n_saved"] == 84# rewrite the token payload into a plain token list, as written by servers that predate the packed server_tokens format- path = os.path.join("tmp", "slot_legacy.bin")+ path = os.path.join(server.slot_save_path, "slot_legacy.bin")with open(path, "rb") as f:data = bytearray(f.read())@@ -462,7 +462,7 @@ def test_slot_save_restore_image_payload_larger_than_context(mmproj_server):})assert res.status_code == 200- path = os.path.join("tmp", "mm_slot_large_payload.bin")+ path = os.path.join(server.slot_save_path, "mm_slot_large_payload.bin")with open(path, "rb") as f:data = bytearray(f.read())payload_size = struct.unpack_from("=I", data, STATE_FILE_HEADER_SIZE - 4)[0]diff --git a/tools/server/tests/unit/test_tool_call.py b/tools/server/tests/unit/test_tool_call.pyindex 9fa84d165e..87c4ad166c 100755--- a/tools/server/tests/unit/test_tool_call.py+++ b/tools/server/tests/unit/test_tool_call.py@@ -21,7 +21,6 @@ def create_server():global serverserver = ServerPreset.tinyllama2()server.model_alias = "tinyllama-2-tool-call"- server.server_port = 8081server.n_slots = 1server.n_ctx = 8192server.n_batch = 2048diff --git a/tools/server/tests/unit/test_tools_builtin.py b/tools/server/tests/unit/test_tools_builtin.pyindex a69052c6d7..d4ebd28d10 100755--- a/tools/server/tests/unit/test_tools_builtin.py+++ b/tools/server/tests/unit/test_tools_builtin.py@@ -64,11 +64,11 @@ def test_tools_builtin_read_file():assert "def test_tools_builtin_read_file" in text-def test_tools_builtin_write_then_edit_file():+def test_tools_builtin_write_then_edit_file(tmp_path):global serverserver.start()- log_path = os.path.join(PROJECT_ROOT, "test.log")+ log_path = str(tmp_path / "test.log")try:write_res = call_tool("write_file", {"path": log_path, "content": "line1\nline2\nline3\n"})assert write_res["result"] == "file written successfully"@@ -93,11 +93,11 @@ def test_tools_builtin_write_then_edit_file():os.remove(log_path)-def test_tools_builtin_edit_file_rejects_non_unique_old_text():+def test_tools_builtin_edit_file_rejects_non_unique_old_text(tmp_path):global serverserver.start()- log_path = os.path.join(PROJECT_ROOT, "test.log")+ log_path = str(tmp_path / "test.log")try:call_tool("write_file", {"path": log_path, "content": "dup\ndup\n"})err = call_tool_expect_error("edit_file", {@@ -275,11 +275,11 @@ def test_tools_builtin_docker_runtime_cleans_up_spawned_container():assert leftover.returncode != 0, f"container {container_id} was not cleaned up after server exit"-def test_tools_builtin_edit_file_rejects_overlapping_edits():+def test_tools_builtin_edit_file_rejects_overlapping_edits(tmp_path):global serverserver.start()- log_path = os.path.join(PROJECT_ROOT, "test.log")+ log_path = str(tmp_path / "test.log")try:call_tool("write_file", {"path": log_path, "content": "line1\nline2\n"})err = call_tool_expect_error("edit_file", {diff --git a/tools/server/tests/utils.py b/tools/server/tests/utils.pyindex 5a4f31a53e..826aef2d5b 100644--- a/tools/server/tests/utils.py+++ b/tools/server/tests/utils.py@@ -294,6 +294,7 @@ class ServerProcess:server_args.append("--backend_sampling")if self.gcp_compat:env["AIP_MODE"] = "PREDICTION"+ env["AIP_HTTP_PORT"] = str(self.server_port)args = [str(arg) for arg in [server_path, *server_args]]print(f"tests: starting server with: {' '.join(args)}")