LLM inference in C/C++
C++ ggml
server : fix usage.completion_tokens undercount with n > 1 (#29486)
Unbug Lee9a3b91c6801e
diff --git a/tools/server/server-context.cpp b/tools/server/server-context.cppindex b8e132d7bf..296893f7a1 100644--- a/tools/server/server-context.cpp+++ b/tools/server/server-context.cpp@@ -5277,10 +5277,20 @@ std::unique_ptr<server_res_generator> server_routes::handle_completions_impl(res->ok(arr[0]);} else if (res_type == TASK_RESPONSE_TYPE_OAI_CHAT || res_type == TASK_RESPONSE_TYPE_OAI_CMPL) {// if multiple results in OAI format, we need to re-format them+ json & usage = arr[0]["usage"];json & choices = arr[0]["choices"];for (size_t i = 1; i < arr.size(); i++) {choices.push_back(std::move(arr[i]["choices"][0]));}++ // OAI counts generated tokens across all choices+ int64_t n_completion_tokens = 0;+ for (auto & item : arr) {+ n_completion_tokens += item["usage"]["completion_tokens"].get<int64_t>();+ }+ usage["completion_tokens"] = n_completion_tokens;+ usage["total_tokens"] = usage["prompt_tokens"].get<int64_t>() + n_completion_tokens;+res->ok(arr[0]);} else {// multi-results, non-OAI compatdiff --git a/tools/server/tests/unit/test_chat_completion.py b/tools/server/tests/unit/test_chat_completion.pyindex 0258b539ed..afbcef0306 100644--- a/tools/server/tests/unit/test_chat_completion.py+++ b/tools/server/tests/unit/test_chat_completion.py@@ -587,6 +587,10 @@ def test_chat_completions_multiple_choices():for choice in res.body["choices"]:assert "assistant" == choice["message"]["role"]assert choice["finish_reason"] == "length"+ # each choice generates max_tokens tokens, usage counts them all+ # ref: https://github.com/ggml-org/llama.cpp/issues/29451+ assert res.body["usage"]["completion_tokens"] == 16+ assert res.body["usage"]["total_tokens"] == res.body["usage"]["prompt_tokens"] + 16def test_chat_completions_token_count():