LLM inference in C/C++
C++ ggml
server: save only restorable slot states to the prompt cache (#30498)
Pascal2639a3b4e029
diff --git a/common/arg.cpp b/common/arg.cppindex 29f6df32fe..8b302d5d77 100644--- a/common/arg.cpp+++ b/common/arg.cpp@@ -1717,10 +1717,11 @@ common_params_context common_params_parser_init(common_params & params, llama_ex).set_env("LLAMA_ARG_CHECKPOINT_MIN_SPACING_NT").set_examples({LLAMA_EXAMPLE_SERVER}));add_opt(common_arg({"-cram", "--cache-ram"}, "N",- string_format("set the maximum cache size in MiB (default: %d, -1 - no limit, 0 - disable)"+ string_format("set the maximum cache size in MiB (default: %d, 0 for decision models, -1 - no limit, 0 - disable)""[(more info)](https://github.com/ggml-org/llama.cpp/pull/16391)", params.cache_ram_mib),[](common_params & params, int value) {params.cache_ram_mib = value;+ params.cache_ram_explicit = true;}).set_env("LLAMA_ARG_CACHE_RAM").set_examples({LLAMA_EXAMPLE_SERVER, LLAMA_EXAMPLE_CLI}));add_opt(common_arg(diff --git a/common/common.h b/common/common.hindex 2048d6a795..2cd6e2ec98 100644--- a/common/common.h+++ b/common/common.h@@ -640,6 +640,7 @@ struct common_params {int32_t kv_unified_per_slot = 0; // max context per parallel slot; 0 = unsetint32_t checkpoint_min_step = 8192; // minimum spacing between context checkpointsint32_t cache_ram_mib = 8192; // -1 = no limit, 0 - disable, 1 = 1 MiB, etc.+ bool cache_ram_explicit = false; // set by --cache-ram, decision models disable the cache otherwisestd::string public_path = ""; // NOLINTstd::string api_prefix = ""; // NOLINTdiff --git a/tools/cli/README.md b/tools/cli/README.mdindex 1637741193..be8fb09d23 100644--- a/tools/cli/README.md+++ b/tools/cli/README.md@@ -148,7 +148,7 @@| `--display-prompt, --no-display-prompt` | whether to print prompt at generation (default: true) || `-co, --color [on\|off\|auto]` | Colorize output to distinguish prompt and user input from generations ('on', 'off', or 'auto', default: 'auto')<br/>'auto' enables colors when output is to a terminal || `-ctxcp, --ctx-checkpoints, --swa-checkpoints N` | max number of context checkpoints to create per slot (default: 32)[(more info)](https://github.com/ggml-org/llama.cpp/pull/15293)<br/>(env: LLAMA_ARG_CTX_CHECKPOINTS) |-| `-cram, --cache-ram N` | set the maximum cache size in MiB (default: 8192, -1 - no limit, 0 - disable)[(more info)](https://github.com/ggml-org/llama.cpp/pull/16391)<br/>(env: LLAMA_ARG_CACHE_RAM) |+| `-cram, --cache-ram N` | set the maximum cache size in MiB (default: 8192, 0 for decision models, -1 - no limit, 0 - disable)[(more info)](https://github.com/ggml-org/llama.cpp/pull/16391)<br/>(env: LLAMA_ARG_CACHE_RAM) || `--context-shift, --no-context-shift` | whether to use context shift on infinite text generation (default: disabled)<br/>(env: LLAMA_ARG_CONTEXT_SHIFT) || `-sys, --system-prompt PROMPT` | system prompt to use with model (if applicable, depending on chat template) || `--show-timings, --no-show-timings` | whether to show timing information after each response (default: true)<br/>(env: LLAMA_ARG_SHOW_TIMINGS) |diff --git a/tools/server/README.md b/tools/server/README.mdindex 37c41f1bb2..4e4472ec24 100644--- a/tools/server/README.md+++ b/tools/server/README.md@@ -165,7 +165,7 @@ For the full list of features, please refer to [server's changelog](https://gith| `--kv-unified-per-slot N` | context limit per parallel slot (default: unset, behavior unchanged).<br/>when set without -c/--ctx-size, the shared KV pool is sized to n_parallel*N<br/>(env: LLAMA_ARG_KV_UNIFIED_PER_SLOT) || `-ctxcp, --ctx-checkpoints, --swa-checkpoints N` | max number of context checkpoints to create per slot (default: 32)[(more info)](https://github.com/ggml-org/llama.cpp/pull/15293)<br/>(env: LLAMA_ARG_CTX_CHECKPOINTS) || `-cms, --checkpoint-min-step N` | minimum spacing between context checkpoints in tokens (default: 8192, 0 = no minimum)<br/>(env: LLAMA_ARG_CHECKPOINT_MIN_SPACING_NT) |-| `-cram, --cache-ram N` | set the maximum cache size in MiB (default: 8192, -1 - no limit, 0 - disable)[(more info)](https://github.com/ggml-org/llama.cpp/pull/16391)<br/>(env: LLAMA_ARG_CACHE_RAM) |+| `-cram, --cache-ram N` | set the maximum cache size in MiB (default: 8192, 0 for decision models, -1 - no limit, 0 - disable)[(more info)](https://github.com/ggml-org/llama.cpp/pull/16391)<br/>(env: LLAMA_ARG_CACHE_RAM) || `-kvu, --kv-unified, -no-kvu, --no-kv-unified` | use single unified KV buffer shared across all sequences (default: enabled if number of slots is auto)<br/>(env: LLAMA_ARG_KV_UNIFIED) || `--cache-idle-slots, --no-cache-idle-slots` | save idle slots to the prompt cache on new task, and clear them when using unified KV (default: enabled, requires cache-ram)<br/>(env: LLAMA_ARG_CACHE_IDLE_SLOTS) || `--context-shift, --no-context-shift` | whether to use context shift on infinite text generation (default: disabled)<br/>(env: LLAMA_ARG_CONTEXT_SHIFT) |diff --git a/tools/server/server-context.cpp b/tools/server/server-context.cppindex 60c5633453..d5b4a7029d 100644--- a/tools/server/server-context.cpp+++ b/tools/server/server-context.cpp@@ -594,7 +594,8 @@ struct server_slot {state = SLOT_STATE_IDLE;// do not keep context of the child slots - the parent's context is enough- if (task->is_child()) {+ // nor of the stateless slots - no task ever reuses it+ if (task->is_child() || task->is_stateless()) {prompt_clear();}@@ -1489,6 +1490,13 @@ private:batch.init(ctx_tgt, std::max(n_batch, params_base.n_parallel), n_embd);}+ // decision requests rarely repeat a state, the prompt cache is opt-in for decision models+ if (decision.type != COMMON_DECISION_TYPE_NONE && !params_base.cache_ram_explicit) {+ SRV_INF("%s", "decision model: prompt cache disabled by default, use `--cache-ram N` to enable it\n");+ params_base.cache_ram_mib = 0;+ params_base.cache_idle_slots = false;+ }+if (params_base.cache_ram_mib != 0) {if (params_base.cache_ram_mib < 0) {SRV_TRC("prompt cache is enabled, size limit: %s\n", "no limit");@@ -1782,8 +1790,8 @@ private:if (ret) {update_cache = update_cache && prompt_cache;- // cache prompts only for completion tasks- update_cache = update_cache && task.type == SERVER_TASK_TYPE_COMPLETION;+ // cache prompts only for the tasks that reuse them+ update_cache = update_cache && !task.is_stateless();if (update_cache) {SRV_TRC("%s", "updating prompt cache\n");@@ -3597,9 +3605,7 @@ private:return;}- const bool is_stateless_task = slot.task->type == SERVER_TASK_TYPE_EMBEDDING || slot.task->type == SERVER_TASK_TYPE_RERANK;-- if (slot.task->params.cache_prompt && !is_stateless_task) {+ if (slot.task->params.cache_prompt && !slot.task->is_stateless()) {// reuse any previously computed tokens that are common with the new promptn_past = slot.prompt.tokens.get_common_prefix(input_tokens);diff --git a/tools/server/server-task.h b/tools/server/server-task.hindex 50926ec3ca..e7bb043988 100644--- a/tools/server/server-task.h+++ b/tools/server/server-task.h@@ -300,6 +300,11 @@ struct server_task {bool is_child() const {return id_parent != -1;}++ // embedding and rerank tasks always process their whole prompt, their state is never reused+ bool is_stateless() const {+ return type == SERVER_TASK_TYPE_EMBEDDING || type == SERVER_TASK_TYPE_RERANK;+ }};struct result_prompt_progress {diff --git a/tools/server/tests/unit/test_rerank.py b/tools/server/tests/unit/test_rerank.pyindex ded8267109..53f15af098 100644--- a/tools/server/tests/unit/test_rerank.py+++ b/tools/server/tests/unit/test_rerank.py@@ -41,6 +41,31 @@ def test_rerank():assert least_relevant["index"] == 3[email protected]("kv_unified", [False, True])+def test_rerank_skips_prompt_cache(kv_unified, tmp_path):+ global server+ server.n_slots = 2+ server.cache_ram = 100+ server.kv_unified = kv_unified+ server.debug = True+ server.log_path = str(tmp_path / "server.log")+ server.start()++ for i in range(4):+ res = server.make_request("POST", "/rerank", data={+ "query": f"query {i}",+ "documents": TEST_DOCUMENTS[i:],+ })+ assert res.status_code == 200++ server.stop()++ with open(server.log_path) as f:+ log = f.read()+ assert "__TEST_TAG_CACHE_IDLE_SLOTS_ENABLED__" in log+ assert "__TEST_TAG_CACHE_IDLE_SLOT__" not in log++def test_rerank_tei_format():global serverserver.start()diff --git a/tools/server/tests/unit/test_systemone.py b/tools/server/tests/unit/test_systemone.pyindex d9d9884105..61c2ab0989 100644--- a/tools/server/tests/unit/test_systemone.py+++ b/tools/server/tests/unit/test_systemone.py@@ -158,6 +158,32 @@ def test_systemone_shared_prompt():assert abs(res.body["answers"]["angry"]["noul"] - res_single.body["answers"]["angry"]["noul"]) < 0.01[email protected]("cache_ram", [None, 100])+def test_systemone_prompt_cache(cache_ram, tmp_path):+ global server+ server.n_slots = 1+ server.cache_ram = cache_ram+ server.debug = True+ server.log_path = str(tmp_path / "server.log")+ server.start()++ # the third request comes back to the state of the first one+ for state in ["Ticket A: " + TEST_STATE, "Ticket B: " + TEST_STATE * 2, "Ticket A: " + TEST_STATE]:+ res = server.make_request("POST", "/v1/systemone", data={+ "state": state,+ "questions": TEST_QUESTIONS,+ })+ assert res.status_code == 200++ server.stop()++ with open(server.log_path) as f:+ log = f.read()+ # the prompt cache is opt-in for decision models, and restores their states once enabled+ assert ("__TEST_TAG_CACHE_IDLE_SLOTS_ENABLED__" in log) == (cache_ram is not None)+ assert ("found better prompt" in log) == (cache_ram is not None)++def test_systemone_images():global serverserver = ServerPreset.tinyopenjev()