LLM inference in C/C++
C++ ggml
server : add trace logging for slot similarity checking (#26271)
Georgi Gerganov992c325323f9
diff --git a/tools/server/server-context.cpp b/tools/server/server-context.cppindex dba15d426f..749bd9aac1 100644--- a/tools/server/server-context.cpp+++ b/tools/server/server-context.cpp@@ -1537,7 +1537,7 @@ private:// find the slot that has at least n% prompt similarityif (slot_prompt_similarity != 0.0f) {- float sim_best = 0;+ float f_sim_best = 0;for (server_slot & slot : slots) {if (task.id_slot != -1 && slot.id != task.id_slot) {@@ -1546,6 +1546,7 @@ private:// skip the slot if it is not availableif (slot.is_processing()) {+ SLT_TRC(slot, " - skipping, is_processing = %d\n", slot.is_processing());continue;}@@ -1553,26 +1554,30 @@ private:// skip the slot if it does not contains cached tokensif (tokens.empty()) {+ SLT_TRC(slot, "%s", " - skipping, slot is empty\n");continue;}// fraction of the Longest Common Prefix length with respect to the input prompt length- const float sim_cur = float(tokens.get_common_prefix(task.tokens)) / task.tokens.size();+ const size_t lcp_len = tokens.get_common_prefix(task.tokens);+ const float f_sim_cur = float(lcp_len) / task.tokens.size();++ SLT_TRC(slot, " - checking sim = %.3f (%zu/%zu) > %.3f\n", f_sim_cur, lcp_len, task.tokens.size(), slot_prompt_similarity);// select the current slot if the criteria match- if (sim_cur > sim_best && sim_cur > slot_prompt_similarity) {- sim_best = sim_cur;+ if (f_sim_cur > f_sim_best && f_sim_cur > slot_prompt_similarity) {+ f_sim_best = f_sim_cur;ret = &slot;}}if (ret != nullptr) {- const float f_keep = (sim_best*task.tokens.size()) / ret->prompt.tokens.size();+ const float f_keep = (f_sim_best*task.tokens.size()) / ret->prompt.tokens.size();if (task.id_slot == -1) {- SLT_INF(*ret, "selected slot by LCP similarity, sim_best = %.3f (> %.3f thold), f_keep = %.3f\n",- sim_best, slot_prompt_similarity, f_keep);+ SLT_INF(*ret, "selected slot by LCP similarity, f_sim_best = %.3f (> %.3f thold), f_keep = %.3f\n",+ f_sim_best, slot_prompt_similarity, f_keep);}// if we are about to lose a large portion of the existing context - save it in the prompt cachediff --git a/tools/server/server-task.cpp b/tools/server/server-task.cppindex 99e63b05f5..070f1ade24 100644--- a/tools/server/server-task.cpp+++ b/tools/server/server-task.cpp@@ -1742,9 +1742,9 @@ bool server_prompt_cache::load(server_prompt & prompt, const server_tokens & tokconst int lcp_best = prompt.tokens.get_common_prefix(tokens_new);float f_keep_best = prompt.tokens.size() > 0 ? float(lcp_best) / prompt.tokens.size() : -1.0f; // empty slot: any cache entry wins- float sim_best = float(lcp_best) / tokens_new.size();+ float f_sim_best = float(lcp_best) / tokens_new.size();- SRV_TRC(" - looking for better prompt, base f_keep = %.3f, sim = %.3f\n", f_keep_best, sim_best);+ SRV_TRC(" - looking for better prompt, base f_keep = %.3f, f_sim = %.3f\n", f_keep_best, f_sim_best);auto it_best = states.end();@@ -1753,25 +1753,25 @@ bool server_prompt_cache::load(server_prompt & prompt, const server_tokens & tokconst int lcp_cur = it->prompt.tokens.get_common_prefix(tokens_new);const float f_keep_cur = float(lcp_cur) / it->prompt.tokens.size();- const float sim_cur = float(lcp_cur) / tokens_new.size();+ const float f_sim_cur = float(lcp_cur) / tokens_new.size();- SRV_TRC(" - prompt with length %7zu, lcp = %7d, f_keep = %.3f, sim = %.3f\n", it->prompt.tokens.size(), lcp_cur, f_keep_cur, sim_cur);+ SRV_TRC(" - prompt with length %7zu, lcp = %7d, f_keep = %.3f, f_sim = %.3f\n", it->prompt.tokens.size(), lcp_cur, f_keep_cur, f_sim_cur);// don't trash large promptsif (f_keep_cur < 0.25f) {continue;}- if (f_keep_best < f_keep_cur && sim_best < sim_cur) {+ if (f_keep_best < f_keep_cur && f_sim_best < f_sim_cur) {f_keep_best = f_keep_cur;- sim_best = sim_cur;+ f_sim_best = f_sim_cur;it_best = it;}}if (it_best != states.end()) {- SRV_TRC(" - found better prompt with f_keep = %.3f, sim = %.3f\n", f_keep_best, sim_best);+ SRV_TRC(" - found better prompt with f_keep = %.3f, f_sim = %.3f\n", f_keep_best, f_sim_best);{auto & data = it_best->data.main;