LLM inference in C/C++
C++ ggml
server: Adding spec-decode counters to /metrics endpoint (#26389)
Bhavik Shardaa035a88878ad
diff --git a/tools/server/README.md b/tools/server/README.mdindex c8e7fcdcf2..4d80f059d5 100644--- a/tools/server/README.md+++ b/tools/server/README.md@@ -1076,6 +1076,10 @@ In *router mode* the query param `?model={model_id}` has to be set. This endpoin| `llamacpp:n_tokens_max` | Counter | High watermark of the context size observed. || `llamacpp:n_decode_total` | Counter | Total Number of llama_decode() calls. || `llamacpp:n_busy_slots_per_decode` | Gauge | Average number of busy slots per llama_decode() call. |+| `llamacpp:spec_decode_num_draft_tokens_total` | Counter | Total draft tokens generated (0 when spec-decode is off). |+| `llamacpp:spec_decode_num_accepted_tokens_total` | Counter | Total draft tokens accepted by the target model (0 when spec-decode is off). |+| `llamacpp:spec_decode_num_drafts_total` | Counter | Total speculative decoding verification steps (0 when spec-decode is off). |+| `llamacpp:spec_decode_num_accepted_tokens_per_pos_total` | Counter | Accepted tokens per draft position (labeled `position="N"`; absent when spec-decode is off or before the first completed speculative request). |### POST `/slots/{id_slot}?action=save`: Save the prompt cache of the specified slot to a file.diff --git a/tools/server/server-context.cpp b/tools/server/server-context.cppindex 380e62af67..38d2e5c7a0 100644--- a/tools/server/server-context.cpp+++ b/tools/server/server-context.cpp@@ -845,6 +845,11 @@ struct server_metrics {uint64_t n_decode_total = 0;uint64_t n_busy_slots_total = 0;+ uint64_t n_draft_tokens_total = 0;+ uint64_t n_draft_accepted_total = 0;+ uint64_t n_draft_verif_steps_total = 0;+ std::vector<uint64_t> n_accepted_per_pos_total;+void init() {t_start = ggml_time_us();}@@ -863,6 +868,17 @@ struct server_metrics {n_tokens_predicted += slot.n_decoded;t_tokens_generation += slot.t_token_generation;t_tokens_generation_total += slot.t_token_generation;++ n_draft_tokens_total += slot.n_draft_total;+ n_draft_accepted_total += slot.n_draft_accepted;+ n_draft_verif_steps_total += slot.n_draft_verif_steps;++ if (n_accepted_per_pos_total.size() < slot.n_accepted_per_pos.size()) {+ n_accepted_per_pos_total.resize(slot.n_accepted_per_pos.size(), 0);+ }+ for (size_t i = 0; i < slot.n_accepted_per_pos.size(); i++) {+ n_accepted_per_pos_total[i] += slot.n_accepted_per_pos[i];+ }}void on_decoded(const std::vector<server_slot> & slots) {@@ -2552,6 +2568,11 @@ private:res->n_decode_total = metrics.n_decode_total;res->n_busy_slots_total = metrics.n_busy_slots_total;+ res->n_draft_tokens_total = metrics.n_draft_tokens_total;+ res->n_draft_accepted_total = metrics.n_draft_accepted_total;+ res->n_draft_verif_steps_total = metrics.n_draft_verif_steps_total;+ res->n_accepted_per_pos_total = metrics.n_accepted_per_pos_total;+if (task.metrics_reset_bucket) {metrics.reset_bucket();}@@ -4440,6 +4461,18 @@ void server_routes::init_routes() {{"name", "n_tokens_max"},{"help", "Largest observed n_tokens."},{"value", res_task->n_tokens_max}+ }, {+ {"name", "spec_decode_num_draft_tokens_total"},+ {"help", "Total draft tokens generated"},+ {"value", res_task->n_draft_tokens_total}+ }, {+ {"name", "spec_decode_num_accepted_tokens_total"},+ {"help", "Total draft tokens accepted by the target model"},+ {"value", res_task->n_draft_accepted_total}+ }, {+ {"name", "spec_decode_num_drafts_total"},+ {"help", "Total speculative decoding verification steps"},+ {"value", res_task->n_draft_verif_steps_total}}}},{"gauge", {{{"name", "prompt_tokens_seconds"},@@ -4481,6 +4514,17 @@ void server_routes::init_routes() {}}+ // labeled counter: one time series per draft position+ if (!res_task->n_accepted_per_pos_total.empty()) {+ prometheus << "# HELP llamacpp:spec_decode_num_accepted_tokens_per_pos_total"+ " Accepted tokens per draft position\n"+ << "# TYPE llamacpp:spec_decode_num_accepted_tokens_per_pos_total counter\n";+ for (size_t i = 0; i < res_task->n_accepted_per_pos_total.size(); i++) {+ prometheus << "llamacpp:spec_decode_num_accepted_tokens_per_pos_total{position=\""+ << i << "\"} " << res_task->n_accepted_per_pos_total[i] << "\n";+ }+ }+res->headers["Process-Start-Time-Unix"] = std::to_string(res_task->t_start);res->content_type = "text/plain; version=0.0.4";res->status = 200;diff --git a/tools/server/server-task.cpp b/tools/server/server-task.cppindex 070f1ade24..1ee6775530 100644--- a/tools/server/server-task.cpp+++ b/tools/server/server-task.cpp@@ -1560,6 +1560,11 @@ json server_task_result_metrics::to_json() {{ "n_decode_total", n_decode_total },{ "n_busy_slots_total", n_busy_slots_total },+ { "n_draft_tokens_total", n_draft_tokens_total },+ { "n_draft_accepted_total", n_draft_accepted_total },+ { "n_draft_verif_steps_total", n_draft_verif_steps_total },+ { "n_accepted_per_pos_total", n_accepted_per_pos_total },+{ "slots", slots_data },};}diff --git a/tools/server/server-task.h b/tools/server/server-task.hindex 411d918079..6275ec7604 100644--- a/tools/server/server-task.h+++ b/tools/server/server-task.h@@ -532,6 +532,11 @@ struct server_task_result_metrics : server_task_result {uint64_t n_decode_total = 0;uint64_t n_busy_slots_total = 0;+ uint64_t n_draft_tokens_total = 0;+ uint64_t n_draft_accepted_total = 0;+ uint64_t n_draft_verif_steps_total = 0;+ std::vector<uint64_t> n_accepted_per_pos_total;+// while we can also use std::vector<server_slot> this requires copying the slot object which can be quite messy// therefore, we use json to temporarily store the slot.to_json() resultjson slots_data = json::array();