LLM inference in C/C++
C++ ggml
tests : update speculative params (#26925)
Georgi Gerganova4a4c51f3d40
diff --git a/tools/server/tests/unit/test_speculative.py b/tools/server/tests/unit/test_speculative.pyindex 0184085b42..5837195006 100644--- a/tools/server/tests/unit/test_speculative.py+++ b/tools/server/tests/unit/test_speculative.py@@ -27,8 +27,8 @@ def test_with_and_without_draft():global serverrequest = {"prompt": "I believe the meaning of life is",- "temperature": 0.8,- "top_k": 40,+ "temperature": 0.2,+ "top_k": 5,"seed": 4242,"n_predict": 16,"return_tokens": True,@@ -36,7 +36,6 @@ def test_with_and_without_draft():server.model_draft = None # disable draft modelserver.spec_type = None- server.backend_sampling = Trueserver.start()res = server.make_request("POST", "/completion", data=request)assert res.status_code == 200@@ -45,7 +44,6 @@ def test_with_and_without_draft():# create new server with draft modelcreate_server()- server.backend_sampling = Trueserver.start()res = server.make_request("POST", "/completion", data=request)assert res.status_code == 200