Skip to content

LLM inference in C/C++

C++ ggml

test : fix multi-GPU server tests (#27614)

Georgi Gerganov8d9af256337d
diff --git a/tools/server/tests/unit/test_slot_save.py b/tools/server/tests/unit/test_slot_save.py
index 05acb1be14..5af61d70d0 100644
--- a/tools/server/tests/unit/test_slot_save.py
+++ b/tools/server/tests/unit/test_slot_save.py
@@ -319,7 +319,6 @@ def test_slot_save_restore_with_two_images(mmproj_server):
"prompt": prompt,
})
assert res.status_code == 200
- content = res.body["content"]
prompt_n_full = res.body["timings"]["prompt_n"]
assert prompt_n_full > 64
@@ -345,6 +344,26 @@ def test_slot_save_restore_with_two_images(mmproj_server):
assert res.status_code == 200
assert res.body["timings"]["cache_n"] == prompt_n_full - 1
assert res.body["timings"]["prompt_n"] == 1
+ content = res.body["content"]
+
+ res = server.make_request("POST", "/slots/1?action=restore", data={
+ "filename": "mm_slot_two_images.bin",
+ })
+ assert res.status_code == 200
+ assert res.body["n_restored"] == n_saved
+
+ res = server.make_request("POST", "/completions", data={
+ "temperature": 0.0,
+ "top_k": 1,
+ "id_slot": 0,
+ "cache_prompt": True,
+ "prompt": prompt,
+ })
+ assert res.status_code == 200
+ assert res.body["timings"]["cache_n"] == prompt_n_full - 1
+ assert res.body["timings"]["prompt_n"] == 1
+ content = res.body["content"]
+
assert res.body["content"] == content
diff --git a/tools/server/tests/unit/test_vision_api.py b/tools/server/tests/unit/test_vision_api.py
index d74cc3a43e..8b01c5372c 100644
--- a/tools/server/tests/unit/test_vision_api.py
+++ b/tools/server/tests/unit/test_vision_api.py
@@ -121,7 +121,7 @@ def test_vision_chat_completion_token_count():
"prompt, image_data, success, re_content",
[
# test model is trained on CIFAR-10, but it's quite dumb due to small size
- ("What is this: <__media__>\n", "IMG_BASE64_0", True, "(cat)+"),
+ ("What is this: <__media__>\n", "IMG_BASE64_0", True, "(cat)+|(automobile)+"),
("What is this: <__media__>\n", "IMG_BASE64_1", True, "(frog)+"),
("What is this: <__media__>\n", "malformed", False, None), # non-image data
("What is this:\n", "", False, None), # empty string
diff --git a/tools/server/tests/utils.py b/tools/server/tests/utils.py
index 9171dbc029..a0d2dfa3c5 100644
--- a/tools/server/tests/utils.py
+++ b/tools/server/tests/utils.py
@@ -623,7 +623,7 @@ class ServerPreset:
server.model_hf_repo = "ggml-org/tinygemma3-GGUF:Q8_0"
server.model_alias = "tinygemma3"
server.n_ctx = 1024
- server.n_batch = 32
+ server.n_batch = 512
server.n_slots = 2
server.n_predict = 4
server.seed = 42