Skip to content

LLM inference in C/C++

C++ ggml

server : optimize get_token_probabilities (#24796)

Adrien Gallouët4b48a53b6cc6
diff --git a/tools/server/server-common.cpp b/tools/server/server-common.cpp
index 75729e62dd..3dc686bb46 100644
--- a/tools/server/server-common.cpp
+++ b/tools/server/server-common.cpp
@@ -12,6 +12,7 @@
#include <random>
#include <sstream>
#include <fstream>
+#include <limits>
json format_error_response(const std::string & message, const enum error_type type) {
std::string type_str;
@@ -1238,7 +1239,7 @@ json format_response_rerank(
// other utils
//
-std::vector<llama_token_data> get_token_probabilities(llama_context * ctx, int idx) {
+std::vector<llama_token_data> get_token_probabilities(llama_context * ctx, int idx, size_t n_top) {
std::vector<llama_token_data> cur;
const auto * logits = llama_get_logits_ith(ctx, idx);
@@ -1257,21 +1258,34 @@ std::vector<llama_token_data> get_token_probabilities(llama_context * ctx, int i
}
}
- // sort tokens by logits
- std::sort(cur.begin(), cur.end(), [](const llama_token_data & a, const llama_token_data & b) {
- return a.logit > b.logit;
- });
+ // sort tokens by logits (partial: only the leading `n_top` need ordering)
+ if (n_top > cur.size()) {
+ n_top = cur.size();
+ }
+ if (n_top > 0) {
+ std::partial_sort(cur.begin(), cur.begin() + n_top, cur.end(),
+ [](const llama_token_data & a, const llama_token_data & b) {
+ return a.logit > b.logit;
+ });
+ }
// apply softmax
- float max_l = cur[0].logit;
+ float max_l = -std::numeric_limits<float>::infinity();
+ if (n_top > 0) {
+ max_l = cur[0].logit; // partial_sort guarantees the absolute maximum is at index 0
+ } else {
+ for (const auto & t : cur) {
+ max_l = std::max(max_l, t.logit);
+ }
+ }
float cum_sum = 0.0f;
- for (size_t i = 0; i < cur.size(); ++i) {
- float p = expf(cur[i].logit - max_l);
- cur[i].p = p;
+ for (auto & t : cur) {
+ float p = expf(t.logit - max_l);
+ t.p = p;
cum_sum += p;
}
- for (size_t i = 0; i < cur.size(); ++i) {
- cur[i].p /= cum_sum;
+ for (auto & t : cur) {
+ t.p /= cum_sum;
}
return cur;
diff --git a/tools/server/server-common.h b/tools/server/server-common.h
index f286b3d156..efd31733b0 100644
--- a/tools/server/server-common.h
+++ b/tools/server/server-common.h
@@ -326,7 +326,7 @@ json format_response_rerank(
// other utils
//
-std::vector<llama_token_data> get_token_probabilities(llama_context * ctx, int idx);
+std::vector<llama_token_data> get_token_probabilities(llama_context * ctx, int idx, size_t n_top);
std::string safe_json_to_str(const json & data);
diff --git a/tools/server/server-context.cpp b/tools/server/server-context.cpp
index 791188b1e7..1f0e1bfd42 100644
--- a/tools/server/server-context.cpp
+++ b/tools/server/server-context.cpp
@@ -1824,8 +1824,7 @@ private:
});
}
} else {
- // TODO: optimize this with min-p optimization
- std::vector<llama_token_data> cur = get_token_probabilities(ctx_tgt, idx);
+ std::vector<llama_token_data> cur = get_token_probabilities(ctx_tgt, idx, n_probs_request);
const size_t max_probs = cur.size();
const size_t n_probs = std::min(max_probs, n_probs_request);