LLM inference in C/C++
C++ ggml
server: fix the off-by-two in the SWA reuse threshold (#30580)
Pascal61ad80bf93de
diff --git a/tools/server/server-context.cpp b/tools/server/server-context.cppindex c20167b90b..a18b7542c3 100644--- a/tools/server/server-context.cpp+++ b/tools/server/server-context.cpp@@ -3784,8 +3784,12 @@ private:// ref: https://github.com/ggml-org/llama.cpp/pull/24110const bool has_new_tokens = (n_past < slot.task->n_tokens());- // the largest pos_min required for a checkpoint to be useful- const auto pos_min_thold = std::max(0, pos_next - n_swa - (has_new_tokens ? 0 : 1));+ // the first evaluated token attends to the positions > pos_eval - n_swa, so a SWA state+ // starting at pos_eval - n_swa + 1 is enough, while a state without SWA must end before pos_eval+ const llama_pos pos_eval = pos_next - (has_new_tokens ? 0 : 1);++ // a state or a checkpoint is useful when its pos_min is below this threshold+ const auto pos_min_thold = std::max(0, n_swa > 0 ? pos_eval - n_swa + 2 : pos_eval);if (n_past > 0 && n_past <= slot.prompt.n_tokens()) {const auto pos_min = llama_memory_seq_pos_min(llama_get_memory(ctx_tgt), slot.id);diff --git a/tools/server/tests/unit/test_slot_save.py b/tools/server/tests/unit/test_slot_save.pyindex 793f1a0a7a..488b63701d 100644--- a/tools/server/tests/unit/test_slot_save.py+++ b/tools/server/tests/unit/test_slot_save.py@@ -838,3 +838,89 @@ def test_slot_restore_checkpoints_draft_kv_type_change(swa_server, ctkd_restore)})assert res.status_code == 200assert res.body["timings"]["prompt_n"] == n_live+++# a SWA window smaller than the prompt, so a saved state holds only the cells inside the window+def swa_window_server(tmp_path):+ swa = ServerPreset.tinygemma3()+ swa.override_kv = "gemma3.attention.sliding_window=int:64"+ swa.slot_save_path = str(tmp_path)+ swa.temperature = 0.0+ swa.n_predict = 8+ return swa+++SWA_PROMPT = "The quick brown fox jumps over the lazy dog. " * 30+++def tokenize(server, text):+ res = server.make_request("POST", "/tokenize", data={"content": text})+ assert res.status_code == 200+ return res.body["tokens"]+++# the first turn and the exact token continuation of it, so the next turn extends the saved state+def swa_turns(server, id_slot):+ res = server.make_request("POST", "/completion", data={+ "prompt": tokenize(server, SWA_PROMPT),+ "id_slot": id_slot,+ "cache_prompt": True,+ "return_tokens": True,+ })+ assert res.status_code == 200+ return tokenize(server, SWA_PROMPT) + res.body["tokens"] + tokenize(server, " The end.")+++def swa_next(server, id_slot, prompt):+ res = server.make_request("POST", "/completion", data={+ "prompt": prompt,+ "id_slot": id_slot,+ "cache_prompt": True,+ })+ assert res.status_code == 200+ return res.body["timings"]+++# a restored SWA state holds the window the next token needs, so it is reused as is+def test_slot_restore_swa_window(swa_window_server):+ server = swa_window_server+ server.cache_ram = 0+ server.start()++ prompt = {id_slot: swa_turns(server, id_slot) for id_slot in [0, 1]}+ assert prompt[0] == prompt[1]++ res = server.make_request("POST", "/slots/0?action=save", data={"filename": "swa_slot0.bin"})+ assert res.status_code == 200+ res = server.make_request("POST", "/slots/0?action=erase")+ assert res.status_code == 200+ res = server.make_request("POST", "/slots/0?action=restore", data={"filename": "swa_slot0.bin"})+ assert res.status_code == 200++ timings = {id_slot: swa_next(server, id_slot, prompt[id_slot]) for id_slot in [0, 1]}++ assert timings[0]["cache_n"] == timings[1]["cache_n"]+ assert timings[0]["prompt_n"] == timings[1]["prompt_n"]+++# a SWA state loaded back from the RAM prompt cache is reused as is+def test_prompt_cache_swa_window(swa_window_server):+ server = swa_window_server+ server.n_slots = 1++ def run(other_prompt_first):+ server.start()+ prompt = swa_turns(server, 0)+ if other_prompt_first:+ res = server.make_request("POST", "/completion", data={"prompt": "Once upon a time " * 60, "cache_prompt": True})+ assert res.status_code == 200+ timings = swa_next(server, 0, prompt)+ server.stop()+ return timings++ direct = run(False)+ cached = run(True)++ assert cached["cache_n"] == direct["cache_n"]+ assert cached["prompt_n"] == direct["prompt_n"]diff --git a/tools/server/tests/utils.py b/tools/server/tests/utils.pyindex b3207d3874..1c84f51211 100644--- a/tools/server/tests/utils.py+++ b/tools/server/tests/utils.py@@ -116,6 +116,7 @@ class ServerProcess:media_path: str | None = Nonesleep_idle_seconds: int | None = Nonecache_ram: int | None = None+ override_kv: str | None = Noneno_cache_idle_slots: bool = Falselog_path: str | None = Noneui_mcp_proxy: bool = False@@ -286,6 +287,8 @@ class ServerProcess:server_args.extend(["--sleep-idle-seconds", self.sleep_idle_seconds])if self.cache_ram is not None:server_args.extend(["--cache-ram", self.cache_ram])+ if self.override_kv:+ server_args.extend(["--override-kv", self.override_kv])if self.no_cache_idle_slots:server_args.append("--no-cache-idle-slots")if self.ui_mcp_proxy: