LLM inference in C/C++
C++ ggml
server : add last-5-seconds generation speed display (#24291)
Aarni Koskela08023072ef63
diff --git a/tools/server/server-context.cpp b/tools/server/server-context.cppindex da6a47586d..31280d63c4 100644--- a/tools/server/server-context.cpp+++ b/tools/server/server-context.cpp@@ -189,9 +189,10 @@ struct server_slot {// statssize_t n_sent_text = 0; // number of sent text character- int64_t t_print_last = 0;int64_t t_start_process_prompt;int64_t t_start_generation;+ int64_t t_print_last = 0;+ int32_t n_decoded_last = 0;double t_prompt_processing = 0.0; // msdouble t_token_generation = 0.0; // ms@@ -470,11 +471,13 @@ struct server_slot {return;}- t_print_last = t_now;+ const double n_gen_second = 1e3 / (t_token_generation) * (n_decoded);+ const double n_gen_second_win = 1e6 / (t_now - t_print_last) * (n_decoded - n_decoded_last);- const double n_gen_second = 1e3 / t_token_generation * n_decoded;+ t_print_last = t_now;+ n_decoded_last = n_decoded;- SLT_INF(*this, "n_decoded = %6d, tg = %6.2f t/s\n", n_decoded, n_gen_second);+ SLT_INF(*this, "n_decoded = %6d, tg = %6.2f t/s, tg_3s = %6.2f t/s\n", n_decoded, n_gen_second, n_gen_second_win);}void print_timings_pp() const {@@ -3038,8 +3041,8 @@ private:}}- const int64_t t_current = ggml_time_us();- slot.t_prompt_processing = (t_current - slot.t_start_process_prompt) / 1e3;+ const int64_t t_now = ggml_time_us();+ slot.t_prompt_processing = (t_now - slot.t_start_process_prompt) / 1e3;slot.print_timings_pp();// truncate any tokens that are beyond n_past for this slot@@ -3447,17 +3450,19 @@ private:common_sampler_accept(slot.smpl.get(), id, true);// here we have synchronized the llama_context (due to the sampling above), so we can do time measurement- const int64_t t_current = ggml_time_us();+ const int64_t t_now = ggml_time_us();slot.n_decoded += 1;if (slot.n_decoded == 1) {- slot.t_start_generation = t_current;+ slot.t_start_generation = t_now;+ slot.t_print_last = t_now;+ slot.n_decoded_last = 0;slot.t_prompt_processing = (slot.t_start_generation - slot.t_start_process_prompt) / 1e3;metrics.on_prompt_eval(slot);}- slot.t_token_generation = std::max<int64_t>(1, t_current - slot.t_start_generation) / 1e3;+ slot.t_token_generation = std::max<int64_t>(1, t_now - slot.t_start_generation) / 1e3;completion_token_output result;result.tok = id;@@ -3551,11 +3556,11 @@ private:slot.spec_draft = std::move(accepted);}- const int64_t t_current = ggml_time_us();+ const int64_t t_now = ggml_time_us();const auto ids = std::move(slot.spec_draft);- slot.t_token_generation = std::max<int64_t>(1, t_current - slot.t_start_generation) / 1e3;+ slot.t_token_generation = std::max<int64_t>(1, t_now - slot.t_start_generation) / 1e3;// update how many tokens out of those tested were acceptedslot.n_draft_accepted += ids.size() - 1;