LLM inference in C/C++
C++ ggml
server : apply checkpoint min-step eviction only when the checkpoint list is full (#28302)
Foad Abo Dahood5d806aa2575e
diff --git a/tools/server/server-context.cpp b/tools/server/server-context.cppindex f78cfb36dd..fe068d3e91 100644--- a/tools/server/server-context.cpp+++ b/tools/server/server-context.cpp@@ -2311,8 +2311,11 @@ private:// evict checkpoints within min-step of a previous checkpoint, unless they were// created by the current task+ // only when the list is full, otherwise short prompts keep just the oldest checkpointint64_t last = -1;- for (auto it = slot.prompt.checkpoints.begin(); it != slot.prompt.checkpoints.end(); ) {+ for (auto it = slot.prompt.checkpoints.begin();+ slot.prompt.checkpoints.size() + 1 >= (size_t) params_base.n_ctx_checkpoints &&+ it != slot.prompt.checkpoints.end(); ) {if (it->id_task != id_task && last >= 0 && it->n_tokens <= last + params_base.checkpoint_min_step) {SLT_TRC(slot, "erasing context checkpoint too close to an earlier one (pos_min = %d, pos_max = %d, n_tokens = %" PRId64 ", size = %.3f MiB)\n",it->pos_min, it->pos_max, it->n_tokens, (float) it->size() / 1024 / 1024);@@ -2335,6 +2338,19 @@ private:slot.prompt.checkpoints.erase(slot.prompt.checkpoints.begin());}+ // replace an existing checkpoint at the same n_tokens instead of appending a duplicate+ {+ const int64_t n_tokens_new = slot.prompt.n_tokens() - n_tokens_cur;+ for (auto it = slot.prompt.checkpoints.begin(); it != slot.prompt.checkpoints.end(); ) {+ if (it->n_tokens == n_tokens_new) {+ SLT_TRC(slot, "superseding context checkpoint at n_tokens = %" PRId64 "\n", it->n_tokens);+ it = slot.prompt.checkpoints.erase(it);+ } else {+ ++it;+ }+ }+ }+auto & cur = slot.prompt.checkpoints.emplace_back();cur.id_task = id_task;