Skip to content

LLM inference in C/C++

C++ ggml

Fix crash with draft-simple (#25720)

Gaurav Garg956973c76466
diff --git a/common/speculative.cpp b/common/speculative.cpp
index 580728a200..3cb08767bd 100644
--- a/common/speculative.cpp
+++ b/common/speculative.cpp
@@ -260,7 +260,10 @@ struct common_speculative_impl_draft_simple : public common_speculative_impl {
bool process(const llama_batch & batch) override {
auto * ctx_dft = params.ctx_dft;
- const int ret = llama_decode(ctx_dft, batch);
+ llama_batch batch_dft = batch;
+ batch_dft.logits = nullptr;
+
+ const int ret = llama_decode(ctx_dft, batch_dft);
if (ret != 0) {
SPC_ERR("failed to decode draft batch, ret = %d\n", ret);
diff --git a/tools/server/tests/unit/test_speculative.py b/tools/server/tests/unit/test_speculative.py
index 84cd77e6f2..c6568479ca 100644
--- a/tools/server/tests/unit/test_speculative.py
+++ b/tools/server/tests/unit/test_speculative.py
@@ -12,8 +12,9 @@ def create_server():
server = ServerPreset.stories15m_moe()
# set default values
server.model_draft = download_file(MODEL_DRAFT_FILE_URL)
- server.draft_min = 4
- server.draft_max = 8
+ server.spec_type = "draft-simple"
+ server.spec_draft_n_min = 4
+ server.spec_draft_n_max = 8
server.fa = "off"
@@ -25,6 +26,7 @@ def fixture_create_server():
def test_with_and_without_draft():
global server
server.model_draft = None # disable draft model
+ server.spec_type = None
server.start()
res = server.make_request("POST", "/completion", data={
"prompt": "I believe the meaning of life is",
@@ -46,6 +48,7 @@ def test_with_and_without_draft():
"n_predict": 16,
})
assert res.status_code == 200
+ assert res.body["timings"]["draft_n"] > 0
content_draft = res.body["content"]
assert content_no_draft == content_draft
@@ -63,8 +66,8 @@ def test_different_draft_min_draft_max():
last_content = None
for draft_min, draft_max in test_values:
server.stop()
- server.draft_min = draft_min
- server.draft_max = draft_max
+ server.spec_draft_n_min = draft_min
+ server.spec_draft_n_max = draft_max
server.start()
res = server.make_request("POST", "/completion", data={
"prompt": "I believe the meaning of life is",
diff --git a/tools/server/tests/utils.py b/tools/server/tests/utils.py
index f4f0e61e61..5d5c873ac4 100644
--- a/tools/server/tests/utils.py
+++ b/tools/server/tests/utils.py
@@ -95,6 +95,7 @@ class ServerProcess:
no_models_autoload: bool | None = None
lora_files: List[str] | None = None
enable_ctx_shift: int | None = False
+ spec_type: str | None = None
spec_draft_n_min: int | None = None
spec_draft_n_max: int | None = None
no_ui: bool | None = None
@@ -226,6 +227,8 @@ class ServerProcess:
server_args.extend(["--lora", lora_file])
if self.enable_ctx_shift:
server_args.append("--context-shift")
+ if self.spec_type:
+ server_args.extend(["--spec-type", self.spec_type])
if self.api_key:
server_args.extend(["--api-key", self.api_key])
if self.spec_draft_n_max: