LLM inference in C/C++
C++ ggml
server : add LLAMA_SERVER_SLOTS_N_DIFF (#27600)
Georgi Gerganove8eed4525aac
diff --git a/tools/server/server-context.cpp b/tools/server/server-context.cppindex 572682af73..a9edbd7be8 100644--- a/tools/server/server-context.cpp+++ b/tools/server/server-context.cpp@@ -858,8 +858,10 @@ private:// slots / clientsstd::vector<server_slot> slots;- int trace = 0;- int slots_debug = 0;+ int trace = 0; // env: LLAMA_TRACE+ int slots_debug = 0; // env: LLAMA_SERVER_SLOTS_DEBUG+ int slots_n_diff = 0; // env: LLAMA_SERVER_SLOTS_N_DIFF+int n_empty_consecutive = 0;std::unique_ptr<server_prompt_cache> prompt_cache;@@ -1247,6 +1249,15 @@ private:}}+ {+ const char * LLAMA_SERVER_SLOTS_N_DIFF = getenv("LLAMA_SERVER_SLOTS_N_DIFF");+ slots_n_diff = LLAMA_SERVER_SLOTS_N_DIFF ? atoi(LLAMA_SERVER_SLOTS_N_DIFF) : 0;++ if (slots_n_diff) {+ SRV_WRN("LLAMA_SERVER_SLOTS_N_DIFF = %d\n", slots_n_diff);+ }+ }+// the update_slots() logic will always submit a maximum of n_batch or n_parallel tokens// note that n_batch can be > n_ctx (e.g. for non-causal attention models such as BERT where the KV cache is not used){@@ -3179,8 +3190,8 @@ private:// when the prompt prefix does not match, print the tokens around the mismatch// this is useful for debugging prompt cachingif (slots_debug) {- const int np0 = std::max<int>(n_past - 4, 0);- const int np1 = std::min<int>(n_past + 6, std::min(slot.prompt.tokens.size(), slot.task->tokens.size()));+ const int np0 = std::max<int>(n_past - slots_n_diff, 0);+ const int np1 = std::min<int>(n_past + slots_n_diff + 2, std::min(slot.prompt.tokens.size(), slot.task->tokens.size()));std::stringstream ss0;std::stringstream ss1;