LLM inference in C/C++
C++ ggml
server: refactor sleep handling, allow access /metrics during sleep (#27376)
Xuan-Son Nguyen947fd9bb2bde
diff --git a/tools/server/README-dev.md b/tools/server/README-dev.mdindex 94fbbde80d..0f42b2ee16 100644--- a/tools/server/README-dev.md+++ b/tools/server/README-dev.md@@ -291,6 +291,36 @@ The flow for downloading a new model:- If a stop request comes in, the router asks the child process to stop (same mechanism as running a model in child process)- Otherwise, upon completion, we call `load_models()` to refresh the list of models+### Sleep mode++Sleep mode was initially introduced in PR [#18228](https://github.com/ggml-org/llama.cpp/pull/18228). The main idea is to have:+- `server_queue` keeping track of the idle timeout+- When the timeout is detected, `server_queue` signals to `server_context_impl` that it should go into sleep+- `server_context_impl` frees all `llama_context` and `mtmd_context`++Compared to simply exiting the whole process, this approach allows accessing some read-only endpoints during sleep, while also handling wakeup-on-request. Any inference request will wake the server up.++Call stack on entering sleeping:+- `server_queue::start_loop` (main thread) sees no task for `idle_sleep_ms` --> `sleeping = true`+- `cb0(true)` --> `server_routes::update_cached_responses`+ - snapshots `/props`, `/models` and metrics; the model is still alive here+- `cb1(true)` --> `server_context_impl::handle_sleeping_state`+ - `callback_state(SERVER_STATE_SLEEPING)` --> reported to router in child mode+ - `destroy()` --> frees `llama_context` and `mtmd_context`+- `condition_tasks.wait` until `req_stop_sleeping`++Call stack on waking up:+- `server_res_generator` constructor (HTTP thread) --> `server_queue::wait_until_no_sleep`+ - sets `req_stop_sleeping = true`, then waits until `sleeping == false`+- `server_queue::start_loop` (main thread) wakes up+- `cb1(false)` --> `server_context_impl::handle_sleeping_state`+ - `load_model()`, which then emits `callback_state(SERVER_STATE_READY)`+- `cb0(false)` --> `server_routes::update_cached_responses`+ - nothing to do, the cache is only read during sleep+- `sleeping = false` --> `notify_all` unblocks the HTTP thread, the request is handled as usual++Endpoints created with `create_response(true)` (`/health`, `/props`, `/models`, `/metrics`) skip `wait_until_no_sleep`, so they answer from the cached responses instead of waking the server.+### Notable Related PRs- Initial server implementation: https://github.com/ggml-org/llama.cpp/pull/1443diff --git a/tools/server/README.md b/tools/server/README.mdindex 78274967db..b63a0e6dac 100644--- a/tools/server/README.md+++ b/tools/server/README.md@@ -2071,6 +2071,7 @@ Note that the following endpoints are exempt from being considered as incoming t- `GET /health`- `GET /props`- `GET /models`+- `GET /metrics`## More examplesdiff --git a/tools/server/server-context.cpp b/tools/server/server-context.cppindex 842e4203cd..21ff783941 100644--- a/tools/server/server-context.cpp+++ b/tools/server/server-context.cpp@@ -818,6 +818,14 @@ public:}}+ server_metrics get_metrics() const {+ return metrics;+ }++ void reset_metrics_bucket() {+ metrics.reset_bucket();+ }+private:// note: accessing these fields outside of this class is not thread-safe// use server_context methods instead@@ -898,6 +906,10 @@ private:void handle_sleeping_state(bool new_state) {GGML_ASSERT(sleeping != new_state);if (new_state) {+ if (callback_state) {+ callback_state(SERVER_STATE_SLEEPING, {});+ // note: for sleeping == false, event is emitted by load_model()+ }SRV_INF("%s", "server is entering sleeping state\n");destroy();} else {@@ -2290,8 +2302,8 @@ private:// returns false to decline the task, it is offered again after the decode is donebool process_single_task(server_task && task, bool is_yielding) {- // while yielding, an encode / decode is running and only accessing metrics is safe- if (is_yielding && task.type != SERVER_TASK_TYPE_METRICS) {+ // while yielding, an encode / decode is running and only reading the server state is safe+ if (is_yielding && task.type != SERVER_TASK_TYPE_METRICS && task.type != SERVER_TASK_TYPE_SLOT_GET) {SRV_DBG("decoding, decline task, id_task = %d\n", task.id);return false;}@@ -2417,28 +2429,17 @@ private:} break;case SERVER_TASK_TYPE_METRICS:{- json slots_data = json::array();-- int n_idle_slots = 0;int n_processing_slots = 0;for (server_slot & slot : slots) {- json slot_data = slot.to_json(slots_debug == 0);-if (slot.is_processing()) {n_processing_slots++;- } else {- n_idle_slots++;}-- slots_data.push_back(slot_data);}- SRV_DBG("n_idle_slots = %d, n_processing_slots = %d\n", n_idle_slots, n_processing_slots);+ SRV_DBG("n_processing_slots = %d\n", n_processing_slots);auto res = std::make_unique<server_task_result_metrics>();res->id = task.id;- res->slots_data = std::move(slots_data);- res->n_idle_slots = n_idle_slots;res->n_processing_slots = n_processing_slots;res->n_tasks_deferred = queue_tasks.queue_tasks_deferred_size();res->metrics = metrics;@@ -2446,6 +2447,28 @@ private:if (task.metrics_reset_bucket) {metrics.reset_bucket();}+ queue_results.send(std::move(res));+ } break;+ case SERVER_TASK_TYPE_SLOT_GET:+ {+ json slots_data = json::array();++ int n_idle_slots = 0;++ for (server_slot & slot : slots) {+ if (!slot.is_processing()) {+ n_idle_slots++;+ }++ slots_data.push_back(slot.to_json(slots_debug == 0));+ }+ SRV_DBG("n_idle_slots = %d\n", n_idle_slots);++ auto res = std::make_unique<server_task_result_slots>();+ res->id = task.id;+ res->slots_data = std::move(slots_data);+ res->n_idle_slots = n_idle_slots;+queue_results.send(std::move(res));} break;case SERVER_TASK_TYPE_SLOT_SAVE:@@ -4142,12 +4165,6 @@ struct server_res_generator : server_res_spipe {void server_context::set_state_callback(server_state_callback_t callback) {impl->callback_state = std::move(callback);- impl->queue_tasks.on_sleeping_state([this](bool sleeping) {- if (sleeping) {- impl->callback_state(SERVER_STATE_SLEEPING, {});- }- // for sleeping == false, event is emitted by load_model()- });}//@@ -4431,6 +4448,119 @@ server_routes::server_routes(const common_params & params, server_context & ctx_queue_tasks(ctx_server.impl->queue_tasks),queue_results(ctx_server.impl->queue_results) {init_routes();++ // note: this must be registered before load_model()+ // so that on sleep phase, the callback is called before ctx is destroyed+ queue_tasks.on_sleeping_state([this](bool is_sleeping) {+ update_cached_responses(is_sleeping);+ });+}++static json get_res_model_info(const server_context_meta & meta) {+ // note: do NOT use ctx_server here, otherwise it's not possible to use this during sleep++ return {+ {"id", meta.model_name},+ {"aliases", meta.model_aliases},+ {"tags", meta.model_tags},+ {"object", "model"},+ {"created", std::time(0)},+ {"owned_by", "llamacpp"},+ {"meta", {+ {"vocab_type", meta.model_vocab_type},+ {"n_vocab", meta.model_vocab_n_tokens},+ {"n_ctx", meta.slot_n_ctx},+ {"n_ctx_train", meta.model_n_ctx_train},+ {"n_embd", meta.model_n_embd_inp},+ {"n_params", meta.model_n_params},+ {"size", meta.model_size},+ {"ftype", meta.model_ftype},+ }},+ };+}++static json get_res_models(const server_context_meta & meta) {+ // note: do NOT use ctx_server here, otherwise it's not possible to use this during sleep++ return {+ {"models", {+ {+ {"name", meta.model_name},+ {"model", meta.model_name},+ {"modified_at", ""},+ {"size", ""},+ {"digest", ""}, // dummy value, llama.cpp does not support managing model file's hash+ {"type", "model"},+ {"description", ""},+ {"tags", {""}},+ {"capabilities", meta.has_mtmd ? json({"completion","multimodal"}) : json({"completion"})},+ {"parameters", ""},+ {"details", {+ {"parent_model", ""},+ {"format", "gguf"},+ {"family", ""},+ {"families", {""}},+ {"parameter_size", ""},+ {"quantization_level", ""}+ }}+ }+ }},+ {"object", "list"},+ {"data", {+ get_res_model_info(meta),+ }}+ };+}++static json get_res_props(const server_context_meta & meta, const common_params & params, bool is_sleeping) {+ // note: do NOT use ctx_server here, otherwise it's not possible to use this during sleep++ task_params tparams;+ tparams.sampling = params.sampling;+ json default_generation_settings_for_props = json {+ { "params", tparams.to_json(true) },+ { "n_ctx", meta.slot_n_ctx },+ };++ std::string tmpl_default = common_chat_templates_source(meta.chat_params.tmpls.get(), "");+ std::string tmpl_tools = common_chat_templates_source(meta.chat_params.tmpls.get(), "tool_use");++ json props = {+ { "default_generation_settings", default_generation_settings_for_props },+ { "total_slots", params.n_parallel },+ { "model_alias", meta.model_name },+ { "model_ftype", meta.model_ftype },+ { "model_path", meta.model_path },+ { "modalities", json {+ {"vision", meta.has_inp_image},+ {"video", meta.has_inp_video},+ {"audio", meta.has_inp_audio},+ } },+ { "media_marker", get_media_marker() },+ { "endpoint_slots", params.endpoint_slots },+ { "endpoint_props", params.endpoint_props },+ { "endpoint_metrics", params.endpoint_metrics },+ { "ui", params.ui },+ { "ui_settings", meta.json_ui_settings },+ { "chat_template", tmpl_default },+ { "chat_template_caps", meta.chat_template_caps },+ { "bos_token", meta.bos_token_str },+ { "eos_token", meta.eos_token_str },+ { "build_info", meta.build_info },+ { "is_sleeping", is_sleeping },+ { "cors_proxy_enabled", params.ui_mcp_proxy },+ };+ if (params.use_jinja) {+ if (!tmpl_tools.empty()) {+ props["chat_template_tool_use"] = tmpl_tools;+ }+ }++ return props;+}++json server_routes::get_model_info() const {+ return get_res_model_info(*meta);}void server_routes::init_routes() {@@ -4451,41 +4581,64 @@ void server_routes::init_routes() {};this->get_metrics = [this](const server_http_req & req) {- auto res = create_response();+ auto res = create_response(true);if (!params.endpoint_metrics) {res->error(format_error_response("This server does not support metrics endpoint. Start it with `--metrics`", ERROR_TYPE_NOT_SUPPORTED));return res;}- // request slots data using task queue- {- server_task task(SERVER_TASK_TYPE_METRICS);- task.id = res->rd.get_new_id();+ // render response using cached_metrics+ auto use_cached_metrics = [&]() {+ std::unique_lock<std::mutex> lock(mutex_cache);+ res->headers["Process-Start-Time-Unix"] = std::to_string(cached_metrics.t_start);+ server_task_result_metrics tmp;+ tmp.metrics = cached_metrics;+ res->content_type = "text/plain; version=0.0.4";+ res->status = 200;+ res->data = tmp.to_metrics();// the gauges are averaged over the window between two scrapes- task.metrics_reset_bucket = true;- res->rd.post_task(std::move(task), true); // high-priority task- }+ cached_metrics.reset_bucket();+ should_reset_buckets = true;+ };- // get the result- auto result = res->rd.next(req.should_stop);- if (!result) {- // connection was closed- GGML_ASSERT(req.should_stop());- return res;- }+ if (queue_tasks.is_sleeping()) {+ use_cached_metrics();- if (result->is_error()) {- res->error(result->to_json());- return res;- }+ } else {+ // request slots data using task queue+ {+ server_task task(SERVER_TASK_TYPE_METRICS);+ task.id = res->rd.get_new_id();+ // the gauges are averaged over the window between two scrapes+ task.metrics_reset_bucket = true;+ res->rd.post_task(std::move(task), true); // high-priority task+ }- auto res_task = dynamic_cast<server_task_result_metrics*>(result.get());- GGML_ASSERT(res_task != nullptr);+ // a task posted right before sleeping is never processed, do not wait for it+ auto result = res->rd.next([&]{+ return req.should_stop() || queue_tasks.is_sleeping();+ });+ if (!result) {+ if (!req.should_stop()) {+ use_cached_metrics();+ }+ return res;+ }++ if (result->is_error()) {+ res->error(result->to_json());+ return res;+ }++ auto res_task = dynamic_cast<server_task_result_metrics*>(result.get());+ GGML_ASSERT(res_task != nullptr);++ res->headers["Process-Start-Time-Unix"] = std::to_string(res_task->metrics.t_start);+ res->content_type = "text/plain; version=0.0.4";+ res->status = 200;+ res->data = res_task->to_metrics();+ }- res->headers["Process-Start-Time-Unix"] = std::to_string(res_task->metrics.t_start);- res->content_type = "text/plain; version=0.0.4";- res->status = 200;- res->data = res_task->to_metrics();return res;};@@ -4498,7 +4651,7 @@ void server_routes::init_routes() {// request slots data using task queue{- server_task task(SERVER_TASK_TYPE_METRICS);+ server_task task(SERVER_TASK_TYPE_SLOT_GET);task.id = res->rd.get_new_id();res->rd.post_task(std::move(task), true); // high-priority task}@@ -4516,7 +4669,7 @@ void server_routes::init_routes() {return res;}- auto * res_task = dynamic_cast<server_task_result_metrics*>(result.get());+ auto * res_task = dynamic_cast<server_task_result_slots*>(result.get());GGML_ASSERT(res_task != nullptr);// optionally return "fail_on_no_slot" error@@ -4566,53 +4719,13 @@ void server_routes::init_routes() {this->get_props = [this](const server_http_req &) {auto res = create_response(true);-- // this endpoint can be accessed during sleeping- // the next LOC is to avoid someone accidentally use ctx_server- bool ctx_server; // do NOT delete this line- GGML_UNUSED(ctx_server);-- task_params tparams;- tparams.sampling = params.sampling;- json default_generation_settings_for_props = json {- { "params", tparams.to_json(true) },- { "n_ctx", meta->slot_n_ctx },- };-- std::string tmpl_default = common_chat_templates_source(meta->chat_params.tmpls.get(), "");- std::string tmpl_tools = common_chat_templates_source(meta->chat_params.tmpls.get(), "tool_use");-- json props = {- { "default_generation_settings", default_generation_settings_for_props },- { "total_slots", params.n_parallel },- { "model_alias", meta->model_name },- { "model_ftype", meta->model_ftype },- { "model_path", meta->model_path },- { "modalities", json {- {"vision", meta->has_inp_image},- {"video", meta->has_inp_video},- {"audio", meta->has_inp_audio},- } },- { "media_marker", get_media_marker() },- { "endpoint_slots", params.endpoint_slots },- { "endpoint_props", params.endpoint_props },- { "endpoint_metrics", params.endpoint_metrics },- { "ui", params.ui },- { "ui_settings", meta->json_ui_settings },- { "chat_template", tmpl_default },- { "chat_template_caps", meta->chat_template_caps },- { "bos_token", meta->bos_token_str },- { "eos_token", meta->eos_token_str },- { "build_info", meta->build_info },- { "is_sleeping", queue_tasks.is_sleeping() },- { "cors_proxy_enabled", params.ui_mcp_proxy },- };- if (params.use_jinja) {- if (!tmpl_tools.empty()) {- props["chat_template_tool_use"] = tmpl_tools;- }+ // note: do NOT use ctx_server here, this endpoint must be accessible during sleep+ if (queue_tasks.is_sleeping()) {+ std::unique_lock<std::mutex> lock(mutex_cache);+ res->ok(cached_props);+ } else {+ res->ok(get_res_props(*meta, params, false));}- res->ok(props);return res;};@@ -4874,42 +4987,13 @@ void server_routes::init_routes() {this->get_models = [this](const server_http_req &) {auto res = create_response(true);-- // this endpoint can be accessed during sleeping- // the next LOC is to avoid someone accidentally use ctx_server- bool ctx_server; // do NOT delete this line- GGML_UNUSED(ctx_server);-- json models = {- {"models", {- {- {"name", meta->model_name},- {"model", meta->model_name},- {"modified_at", ""},- {"size", ""},- {"digest", ""}, // dummy value, llama.cpp does not support managing model file's hash- {"type", "model"},- {"description", ""},- {"tags", {""}},- {"capabilities", meta->has_mtmd ? json({"completion","multimodal"}) : json({"completion"})},- {"parameters", ""},- {"details", {- {"parent_model", ""},- {"format", "gguf"},- {"family", ""},- {"families", {""}},- {"parameter_size", ""},- {"quantization_level", ""}- }}- }- }},- {"object", "list"},- {"data", {- get_model_info(),- }}- };-- res->ok(models);+ // note: do NOT use ctx_server here, this endpoint must be accessible during sleep+ if (queue_tasks.is_sleeping()) {+ std::unique_lock<std::mutex> lock(mutex_cache);+ res->ok(cached_models);+ } else {+ res->ok(get_res_models(*meta));+ }return res;};@@ -5119,27 +5203,6 @@ void server_routes::init_routes() {};}-json server_routes::get_model_info() const {- return json {- {"id", meta->model_name},- {"aliases", meta->model_aliases},- {"tags", meta->model_tags},- {"object", "model"},- {"created", std::time(0)},- {"owned_by", "llamacpp"},- {"meta", {- {"vocab_type", meta->model_vocab_type},- {"n_vocab", meta->model_vocab_n_tokens},- {"n_ctx", meta->slot_n_ctx},- {"n_ctx_train", meta->model_n_ctx_train},- {"n_embd", meta->model_n_embd_inp},- {"n_params", meta->model_n_params},- {"size", meta->model_size},- {"ftype", meta->model_ftype},- }},- };-}-std::unique_ptr<server_res_generator> server_routes::handle_slots_save(const server_http_req & req, int id_slot) {auto res = create_response();const json request_data = json::parse(req.body);@@ -5388,3 +5451,24 @@ std::unique_ptr<server_res_generator> server_routes::handle_count_tokens(const lres->ok(response);return res;}++void server_routes::update_cached_responses(bool is_sleeping) {+ // caller is task_queue, so ctx_server can be accessed without holding locks+ std::unique_lock<std::mutex> lock(mutex_cache);++ if (is_sleeping) {+ cached_models = get_res_models(*meta);+ cached_props = get_res_props(*meta, params, true);+ cached_metrics = ctx_server.get_metrics();++ should_reset_buckets = false;++ SRV_DBG("%s\n", "cached responses updated");++ } else if (should_reset_buckets) {+ // a scrape during sleep already reported these buckets+ ctx_server.reset_metrics_bucket();++ should_reset_buckets = false;+ }+}diff --git a/tools/server/server-context.h b/tools/server/server-context.hindex f9ab1132b1..764df0e085 100644--- a/tools/server/server-context.h+++ b/tools/server/server-context.h@@ -8,6 +8,7 @@#include <cstddef>#include <memory>+#include <mutex>#include <set>struct server_context_impl; // private implementation@@ -174,9 +175,19 @@ private:std::unique_ptr<const server_context_meta> meta;const common_params & params;- const server_context_impl & ctx_server;+ server_context_impl & ctx_server;server_queue & queue_tasks;server_response & queue_results;std::unique_ptr<server_res_generator> create_response(bool bypass_sleep = false);++ // cached responses, to be used during sleep+ std::mutex mutex_cache;+ json cached_models = nullptr;+ json cached_props = nullptr;+ server_metrics cached_metrics;+ // set when a scrape during sleep already reported the throughput buckets+ bool should_reset_buckets = false;+ // call right before sleep to update the cached responses+ void update_cached_responses(bool is_sleeping);};diff --git a/tools/server/server-queue.cpp b/tools/server/server-queue.cppindex 2bcc9bd8f2..78169e9a5d 100644--- a/tools/server/server-queue.cpp+++ b/tools/server/server-queue.cpp@@ -3,6 +3,7 @@#include "log.h"+#include <algorithm>#include <chrono>#include <thread>@@ -20,6 +21,10 @@// server_queue//+static bool task_resets_idle_timer(server_task_type type) {+ return type != SERVER_TASK_TYPE_METRICS;+}+int server_queue::post(server_task && task, bool front) {std::unique_lock<std::mutex> lock(mutex_tasks);GGML_ASSERT(task.id != -1);@@ -27,20 +32,24 @@ int server_queue::post(server_task && task, bool front) {if (task.type == SERVER_TASK_TYPE_CANCEL) {cleanup_pending_task(task.id_target);}- const int task_id = task.id;+ const int task_id = task.id;+ const bool reset_timer = task_resets_idle_timer(task.type);QUE_DBG("new task, id = %d, front = %d\n", task_id, front);if (front) {queue_tasks.push_front(std::move(task));} else {queue_tasks.push_back(std::move(task));}- time_last_task = ggml_time_ms();+ if (reset_timer) {+ time_last_task = ggml_time_ms();+ }condition_tasks.notify_one();return task_id;}int server_queue::post(std::vector<server_task> && tasks, bool front) {std::unique_lock<std::mutex> lock(mutex_tasks);+ bool reset_timer = false;for (auto & task : tasks) {if (task.id == -1) {task.id = id++;@@ -49,6 +58,7 @@ int server_queue::post(std::vector<server_task> && tasks, bool front) {if (task.type == SERVER_TASK_TYPE_CANCEL) {cleanup_pending_task(task.id_target);}+ reset_timer |= task_resets_idle_timer(task.type);QUE_DBG("new task, id = %d/%d, front = %d\n", task.id, (int) tasks.size(), front);if (front) {queue_tasks.push_front(std::move(task));@@ -56,7 +66,9 @@ int server_queue::post(std::vector<server_task> && tasks, bool front) {queue_tasks.push_back(std::move(task));}}- time_last_task = ggml_time_ms();+ if (reset_timer) {+ time_last_task = ggml_time_ms();+ }condition_tasks.notify_one();return 0;}@@ -294,11 +306,14 @@ void server_queue::start_loop(int64_t idle_sleep_ms) {QUE_DBG("%s", "update slots\n");// this will run the main inference process for all slots+ const int64_t t_update_slots = ggml_time_ms();callback_update_slots();{// update_slots() may take a while to finish, we need to make sure it's not counted as idle+ // shift instead of reset, so that non-task_resets_idle_timer tasks do not delay the sleepstd::unique_lock<std::mutex> lock(mutex_tasks);- time_last_task = ggml_time_ms();+ const int64_t now = ggml_time_ms();+ time_last_task = std::min(now, time_last_task + (now - t_update_slots));}QUE_DBG("%s", "waiting for new tasks\n");@@ -312,7 +327,10 @@ void server_queue::start_loop(int64_t idle_sleep_ms) {if (should_sleep()) {QUE_INF("%s", "entering sleeping state\n");sleeping = true;- callback_sleeping_state(true);+ // Call order cb0 -> cb1 -> cb{N}+ for (auto & cb : callback_sleeping_state) {+ cb(true);+ }req_stop_sleeping = false;// wait until we are requested to exit sleeping statecondition_tasks.wait(lock, [&]{@@ -323,7 +341,10 @@ void server_queue::start_loop(int64_t idle_sleep_ms) {}QUE_INF("%s", "exiting sleeping state\n");req_stop_sleeping = false;- callback_sleeping_state(false);+ // Call order cb{N} -> cb1 -> cb0+ for (size_t i = callback_sleeping_state.size(); i > 0; i--) {+ callback_sleeping_state[i - 1](false);+ }sleeping = false;time_last_task = ggml_time_ms();condition_tasks.notify_all(); // notify wait_until_no_sleep()diff --git a/tools/server/server-queue.h b/tools/server/server-queue.hindex 52d30095c1..e17733a743 100644--- a/tools/server/server-queue.h+++ b/tools/server/server-queue.h@@ -44,7 +44,7 @@ private:// callback functionsstd::function<bool(server_task &&, bool)> callback_new_task;std::function<void(void)> callback_update_slots;- std::function<void(bool)> callback_sleeping_state;+ std::vector<std::function<void(bool)>> callback_sleeping_state;public:~server_queue() { worker_stop(); }@@ -86,6 +86,7 @@ public:** Sleeping procedure (disabled if idle_sleep_ms < 0):* - If there is no task after idle_sleep_ms, enter sleeping state+ * note: metrics tasks are processed as usual, but do not reset the idle timer* - Call callback_sleeping_state(true)* - Wait until req_stop_sleeping is set to true* - Call callback_sleeping_state(false)@@ -127,18 +128,12 @@ public:}// Register callback for sleeping state change; multiple callbacks are allowed- // note: when entering sleeping state, the callback is called AFTER sleeping is set to true- // when leaving sleeping state, the callback is called BEFORE sleeping is set to false+ // for example: register order cb0, cb1, cb2+ // entering sleep: queue.sleeping = true --> cb0(true) --> cb1(true) --> cb2(true)+ // leaving sleep: cb2(false) --> cb1(false) --> cb0(false) --> queue.sleeping = false+ // note: caller will hold mutex_tasks while calling the callbacksvoid on_sleeping_state(std::function<void(bool)> callback) {- if (callback_sleeping_state) {- auto prev_callback = std::move(callback_sleeping_state);- callback_sleeping_state = [prev_callback, callback](bool sleeping) {- prev_callback(sleeping);- callback(sleeping);- };- } else {- callback_sleeping_state = std::move(callback);- }+ callback_sleeping_state.push_back(std::move(callback));}private:diff --git a/tools/server/server-task.cpp b/tools/server/server-task.cppindex 64afbc5edf..258cdcf8fb 100644--- a/tools/server/server-task.cpp+++ b/tools/server/server-task.cpp@@ -1512,10 +1512,15 @@ json server_task_result_error::to_json() {//// server_task_result_metrics//-json server_task_result_metrics::to_json() {+json server_task_result_slots::to_json() {return slots_data;}+json server_task_result_metrics::to_json() {+ // not used, /metrics renders prometheus text via to_metrics()+ return json{};+}+// metrics definition: https://prometheus.io/docs/practices/naming/#metric-namesstd::string server_task_result_metrics::to_metrics() {const std::vector<metric_item> counters = {diff --git a/tools/server/server-task.h b/tools/server/server-task.hindex b6da4d4bd6..25ff015122 100644--- a/tools/server/server-task.h+++ b/tools/server/server-task.h@@ -22,6 +22,7 @@ enum server_task_type {SERVER_TASK_TYPE_CONTROL,SERVER_TASK_TYPE_NEXT_RESPONSE,SERVER_TASK_TYPE_METRICS,+ SERVER_TASK_TYPE_SLOT_GET,SERVER_TASK_TYPE_SLOT_SAVE,SERVER_TASK_TYPE_SLOT_RESTORE,SERVER_TASK_TYPE_SLOT_ERASE,@@ -489,22 +490,16 @@ struct server_task_result_error : server_task_result {virtual json to_json() override;};+// used by /metrics APIstruct server_task_result_metrics : server_task_result {// these are immediate stats, not accumulated (server_metrics is cumulative)- int n_idle_slots;- int n_processing_slots;- int n_tasks_deferred;+ int n_processing_slots = 0;+ int n_tasks_deferred = 0;server_metrics metrics;- // while we can also use std::vector<server_slot> this requires copying the slot object which can be quite messy- // therefore, we use json to temporarily store the slot.to_json() result- json slots_data = json::array();-- // used by /slots APIvirtual json to_json() override;- // used by /metrics APIstruct metric_item {std::string name;std::string description;@@ -513,6 +508,17 @@ struct server_task_result_metrics : server_task_result {std::string to_metrics();};+// used by /slots API+struct server_task_result_slots : server_task_result {+ int n_idle_slots = 0;++ // while we can also use std::vector<server_slot> this requires copying the slot object which can be quite messy+ // therefore, we use json to temporarily store the slot.to_json() result+ json slots_data = json::array();++ virtual json to_json() override;+};+struct server_task_result_slot_save_load : server_task_result {std::string filename;bool is_save; // true = save, false = loaddiff --git a/tools/server/tests/unit/test_sleep.py b/tools/server/tests/unit/test_sleep.pyindex 3374165e83..515f7077d3 100644--- a/tools/server/tests/unit/test_sleep.py+++ b/tools/server/tests/unit/test_sleep.py@@ -11,6 +11,35 @@ def create_server():server = ServerPreset.tinyllama2()+def is_sleeping(server: ServerProcess) -> bool:+ res = server.make_request("GET", "/props")+ assert res.status_code == 200+ return res.body["is_sleeping"]+++def wait_for_sleep(server: ServerProcess, timeout: float = 10.0):+ start = time.time()+ while time.time() - start < timeout:+ if is_sleeping(server):+ return+ time.sleep(0.1)+ raise TimeoutError("server did not go to sleep")+++def fetch_metrics(server: ServerProcess) -> str:+ res = server.make_request("GET", "/metrics")+ assert res.status_code == 200+ assert isinstance(res.body, str)+ return res.body+++def get_metric(text: str, name: str) -> float:+ prefix = f"llamacpp:{name} "+ values = [ln for ln in text.splitlines() if ln.startswith(prefix)]+ assert len(values) == 1, f"{name} not found in metrics"+ return float(values[0][len(prefix):])++def test_server_sleep():global serverserver.sleep_idle_seconds = 1@@ -25,6 +54,10 @@ def test_server_sleep():res = server.make_request("GET", "/props")assert res.status_code == 200assert res.body["is_sleeping"] == True+ res = server.make_request("GET", "/models")+ assert res.status_code == 200+ assert len(res.body["data"]) == 1+ assert res.body["data"][0]["id"] == server.model_alias# make a generation request to wake up the serverres = server.make_request("POST", "/completion", data={@@ -37,3 +70,58 @@ def test_server_sleep():res = server.make_request("GET", "/props")assert res.status_code == 200assert res.body["is_sleeping"] == False+++def test_server_sleep_read_only_endpoints():+ global server+ server.sleep_idle_seconds = 1+ server.server_metrics = True+ server.start()++ res = server.make_request("POST", "/completion", data={+ "n_predict": 4,+ "prompt": "Hello",+ })+ assert res.status_code == 200++ # the first scrape resets the throughput buckets, so that the second one reports+ # the same zero rates as the snapshot taken on entering sleep+ fetch_metrics(server)+ metrics_awake = fetch_metrics(server)+ assert get_metric(metrics_awake, "tokens_predicted_total") > 0++ wait_for_sleep(server)++ # during sleep, metrics are served from the snapshot taken right before sleeping+ assert fetch_metrics(server) == metrics_awake++ # scraping /metrics must not wake the server up+ assert is_sleeping(server)+++def test_server_sleep_metrics_buckets():+ global server+ server.sleep_idle_seconds = 1+ server.server_metrics = True+ server.start()++ res = server.make_request("POST", "/completion", data={+ "n_predict": 8,+ "prompt": "Hello",+ })+ assert res.status_code == 200++ wait_for_sleep(server)++ # the first scrape reports the throughput of the last generation+ assert get_metric(fetch_metrics(server), "predicted_tokens_seconds") > 0++ # nothing runs while sleeping, so the next scrapes report an empty window+ assert get_metric(fetch_metrics(server), "predicted_tokens_seconds") == 0+ assert is_sleeping(server)++ # waking up must not report the buckets again+ res = server.make_request("POST", "/tokenize", data={"content": "Hello"})+ assert res.status_code == 200+ assert is_sleeping(server) == False+ assert get_metric(fetch_metrics(server), "predicted_tokens_seconds") == 0