LLM inference in C/C++
C++ ggml
Fix crash with draft-simple (#25720)
Gaurav Garg956973c76466
diff --git a/common/speculative.cpp b/common/speculative.cppindex 580728a200..3cb08767bd 100644--- a/common/speculative.cpp+++ b/common/speculative.cpp@@ -260,7 +260,10 @@ struct common_speculative_impl_draft_simple : public common_speculative_impl {bool process(const llama_batch & batch) override {auto * ctx_dft = params.ctx_dft;- const int ret = llama_decode(ctx_dft, batch);+ llama_batch batch_dft = batch;+ batch_dft.logits = nullptr;++ const int ret = llama_decode(ctx_dft, batch_dft);if (ret != 0) {SPC_ERR("failed to decode draft batch, ret = %d\n", ret);diff --git a/tools/server/tests/unit/test_speculative.py b/tools/server/tests/unit/test_speculative.pyindex 84cd77e6f2..c6568479ca 100644--- a/tools/server/tests/unit/test_speculative.py+++ b/tools/server/tests/unit/test_speculative.py@@ -12,8 +12,9 @@ def create_server():server = ServerPreset.stories15m_moe()# set default valuesserver.model_draft = download_file(MODEL_DRAFT_FILE_URL)- server.draft_min = 4- server.draft_max = 8+ server.spec_type = "draft-simple"+ server.spec_draft_n_min = 4+ server.spec_draft_n_max = 8server.fa = "off"@@ -25,6 +26,7 @@ def fixture_create_server():def test_with_and_without_draft():global serverserver.model_draft = None # disable draft model+ server.spec_type = Noneserver.start()res = server.make_request("POST", "/completion", data={"prompt": "I believe the meaning of life is",@@ -46,6 +48,7 @@ def test_with_and_without_draft():"n_predict": 16,})assert res.status_code == 200+ assert res.body["timings"]["draft_n"] > 0content_draft = res.body["content"]assert content_no_draft == content_draft@@ -63,8 +66,8 @@ def test_different_draft_min_draft_max():last_content = Nonefor draft_min, draft_max in test_values:server.stop()- server.draft_min = draft_min- server.draft_max = draft_max+ server.spec_draft_n_min = draft_min+ server.spec_draft_n_max = draft_maxserver.start()res = server.make_request("POST", "/completion", data={"prompt": "I believe the meaning of life is",diff --git a/tools/server/tests/utils.py b/tools/server/tests/utils.pyindex f4f0e61e61..5d5c873ac4 100644--- a/tools/server/tests/utils.py+++ b/tools/server/tests/utils.py@@ -95,6 +95,7 @@ class ServerProcess:no_models_autoload: bool | None = Nonelora_files: List[str] | None = Noneenable_ctx_shift: int | None = False+ spec_type: str | None = Nonespec_draft_n_min: int | None = Nonespec_draft_n_max: int | None = Noneno_ui: bool | None = None@@ -226,6 +227,8 @@ class ServerProcess:server_args.extend(["--lora", lora_file])if self.enable_ctx_shift:server_args.append("--context-shift")+ if self.spec_type:+ server_args.extend(["--spec-type", self.spec_type])if self.api_key:server_args.extend(["--api-key", self.api_key])if self.spec_draft_n_max: