LLM inference in C/C++
C++ ggml
test : fix multi-GPU server tests (#27614)
Georgi Gerganov8d9af256337d
diff --git a/tools/server/tests/unit/test_slot_save.py b/tools/server/tests/unit/test_slot_save.pyindex 05acb1be14..5af61d70d0 100644--- a/tools/server/tests/unit/test_slot_save.py+++ b/tools/server/tests/unit/test_slot_save.py@@ -319,7 +319,6 @@ def test_slot_save_restore_with_two_images(mmproj_server):"prompt": prompt,})assert res.status_code == 200- content = res.body["content"]prompt_n_full = res.body["timings"]["prompt_n"]assert prompt_n_full > 64@@ -345,6 +344,26 @@ def test_slot_save_restore_with_two_images(mmproj_server):assert res.status_code == 200assert res.body["timings"]["cache_n"] == prompt_n_full - 1assert res.body["timings"]["prompt_n"] == 1+ content = res.body["content"]++ res = server.make_request("POST", "/slots/1?action=restore", data={+ "filename": "mm_slot_two_images.bin",+ })+ assert res.status_code == 200+ assert res.body["n_restored"] == n_saved++ res = server.make_request("POST", "/completions", data={+ "temperature": 0.0,+ "top_k": 1,+ "id_slot": 0,+ "cache_prompt": True,+ "prompt": prompt,+ })+ assert res.status_code == 200+ assert res.body["timings"]["cache_n"] == prompt_n_full - 1+ assert res.body["timings"]["prompt_n"] == 1+ content = res.body["content"]+assert res.body["content"] == contentdiff --git a/tools/server/tests/unit/test_vision_api.py b/tools/server/tests/unit/test_vision_api.pyindex d74cc3a43e..8b01c5372c 100644--- a/tools/server/tests/unit/test_vision_api.py+++ b/tools/server/tests/unit/test_vision_api.py@@ -121,7 +121,7 @@ def test_vision_chat_completion_token_count():"prompt, image_data, success, re_content",[# test model is trained on CIFAR-10, but it's quite dumb due to small size- ("What is this: <__media__>\n", "IMG_BASE64_0", True, "(cat)+"),+ ("What is this: <__media__>\n", "IMG_BASE64_0", True, "(cat)+|(automobile)+"),("What is this: <__media__>\n", "IMG_BASE64_1", True, "(frog)+"),("What is this: <__media__>\n", "malformed", False, None), # non-image data("What is this:\n", "", False, None), # empty stringdiff --git a/tools/server/tests/utils.py b/tools/server/tests/utils.pyindex 9171dbc029..a0d2dfa3c5 100644--- a/tools/server/tests/utils.py+++ b/tools/server/tests/utils.py@@ -623,7 +623,7 @@ class ServerPreset:server.model_hf_repo = "ggml-org/tinygemma3-GGUF:Q8_0"server.model_alias = "tinygemma3"server.n_ctx = 1024- server.n_batch = 32+ server.n_batch = 512server.n_slots = 2server.n_predict = 4server.seed = 42