Skip to content

LLM inference in C/C++

C++ ggml

server: refactor sleep handling, allow access /metrics during sleep (#27376)

Xuan-Son Nguyen947fd9bb2bde
diff --git a/tools/server/README-dev.md b/tools/server/README-dev.md
index 94fbbde80d..0f42b2ee16 100644
--- a/tools/server/README-dev.md
+++ b/tools/server/README-dev.md
@@ -291,6 +291,36 @@ The flow for downloading a new model:
- If a stop request comes in, the router asks the child process to stop (same mechanism as running a model in child process)
- Otherwise, upon completion, we call `load_models()` to refresh the list of models
+### Sleep mode
+
+Sleep mode was initially introduced in PR [#18228](https://github.com/ggml-org/llama.cpp/pull/18228). The main idea is to have:
+- `server_queue` keeping track of the idle timeout
+- When the timeout is detected, `server_queue` signals to `server_context_impl` that it should go into sleep
+- `server_context_impl` frees all `llama_context` and `mtmd_context`
+
+Compared to simply exiting the whole process, this approach allows accessing some read-only endpoints during sleep, while also handling wakeup-on-request. Any inference request will wake the server up.
+
+Call stack on entering sleeping:
+- `server_queue::start_loop` (main thread) sees no task for `idle_sleep_ms` --> `sleeping = true`
+- `cb0(true)` --> `server_routes::update_cached_responses`
+ - snapshots `/props`, `/models` and metrics; the model is still alive here
+- `cb1(true)` --> `server_context_impl::handle_sleeping_state`
+ - `callback_state(SERVER_STATE_SLEEPING)` --> reported to router in child mode
+ - `destroy()` --> frees `llama_context` and `mtmd_context`
+- `condition_tasks.wait` until `req_stop_sleeping`
+
+Call stack on waking up:
+- `server_res_generator` constructor (HTTP thread) --> `server_queue::wait_until_no_sleep`
+ - sets `req_stop_sleeping = true`, then waits until `sleeping == false`
+- `server_queue::start_loop` (main thread) wakes up
+- `cb1(false)` --> `server_context_impl::handle_sleeping_state`
+ - `load_model()`, which then emits `callback_state(SERVER_STATE_READY)`
+- `cb0(false)` --> `server_routes::update_cached_responses`
+ - nothing to do, the cache is only read during sleep
+- `sleeping = false` --> `notify_all` unblocks the HTTP thread, the request is handled as usual
+
+Endpoints created with `create_response(true)` (`/health`, `/props`, `/models`, `/metrics`) skip `wait_until_no_sleep`, so they answer from the cached responses instead of waking the server.
+
### Notable Related PRs
- Initial server implementation: https://github.com/ggml-org/llama.cpp/pull/1443
diff --git a/tools/server/README.md b/tools/server/README.md
index 78274967db..b63a0e6dac 100644
--- a/tools/server/README.md
+++ b/tools/server/README.md
@@ -2071,6 +2071,7 @@ Note that the following endpoints are exempt from being considered as incoming t
- `GET /health`
- `GET /props`
- `GET /models`
+- `GET /metrics`
## More examples
diff --git a/tools/server/server-context.cpp b/tools/server/server-context.cpp
index 842e4203cd..21ff783941 100644
--- a/tools/server/server-context.cpp
+++ b/tools/server/server-context.cpp
@@ -818,6 +818,14 @@ public:
}
}
+ server_metrics get_metrics() const {
+ return metrics;
+ }
+
+ void reset_metrics_bucket() {
+ metrics.reset_bucket();
+ }
+
private:
// note: accessing these fields outside of this class is not thread-safe
// use server_context methods instead
@@ -898,6 +906,10 @@ private:
void handle_sleeping_state(bool new_state) {
GGML_ASSERT(sleeping != new_state);
if (new_state) {
+ if (callback_state) {
+ callback_state(SERVER_STATE_SLEEPING, {});
+ // note: for sleeping == false, event is emitted by load_model()
+ }
SRV_INF("%s", "server is entering sleeping state\n");
destroy();
} else {
@@ -2290,8 +2302,8 @@ private:
// returns false to decline the task, it is offered again after the decode is done
bool process_single_task(server_task && task, bool is_yielding) {
- // while yielding, an encode / decode is running and only accessing metrics is safe
- if (is_yielding && task.type != SERVER_TASK_TYPE_METRICS) {
+ // while yielding, an encode / decode is running and only reading the server state is safe
+ if (is_yielding && task.type != SERVER_TASK_TYPE_METRICS && task.type != SERVER_TASK_TYPE_SLOT_GET) {
SRV_DBG("decoding, decline task, id_task = %d\n", task.id);
return false;
}
@@ -2417,28 +2429,17 @@ private:
} break;
case SERVER_TASK_TYPE_METRICS:
{
- json slots_data = json::array();
-
- int n_idle_slots = 0;
int n_processing_slots = 0;
for (server_slot & slot : slots) {
- json slot_data = slot.to_json(slots_debug == 0);
-
if (slot.is_processing()) {
n_processing_slots++;
- } else {
- n_idle_slots++;
}
-
- slots_data.push_back(slot_data);
}
- SRV_DBG("n_idle_slots = %d, n_processing_slots = %d\n", n_idle_slots, n_processing_slots);
+ SRV_DBG("n_processing_slots = %d\n", n_processing_slots);
auto res = std::make_unique<server_task_result_metrics>();
res->id = task.id;
- res->slots_data = std::move(slots_data);
- res->n_idle_slots = n_idle_slots;
res->n_processing_slots = n_processing_slots;
res->n_tasks_deferred = queue_tasks.queue_tasks_deferred_size();
res->metrics = metrics;
@@ -2446,6 +2447,28 @@ private:
if (task.metrics_reset_bucket) {
metrics.reset_bucket();
}
+ queue_results.send(std::move(res));
+ } break;
+ case SERVER_TASK_TYPE_SLOT_GET:
+ {
+ json slots_data = json::array();
+
+ int n_idle_slots = 0;
+
+ for (server_slot & slot : slots) {
+ if (!slot.is_processing()) {
+ n_idle_slots++;
+ }
+
+ slots_data.push_back(slot.to_json(slots_debug == 0));
+ }
+ SRV_DBG("n_idle_slots = %d\n", n_idle_slots);
+
+ auto res = std::make_unique<server_task_result_slots>();
+ res->id = task.id;
+ res->slots_data = std::move(slots_data);
+ res->n_idle_slots = n_idle_slots;
+
queue_results.send(std::move(res));
} break;
case SERVER_TASK_TYPE_SLOT_SAVE:
@@ -4142,12 +4165,6 @@ struct server_res_generator : server_res_spipe {
void server_context::set_state_callback(server_state_callback_t callback) {
impl->callback_state = std::move(callback);
- impl->queue_tasks.on_sleeping_state([this](bool sleeping) {
- if (sleeping) {
- impl->callback_state(SERVER_STATE_SLEEPING, {});
- }
- // for sleeping == false, event is emitted by load_model()
- });
}
//
@@ -4431,6 +4448,119 @@ server_routes::server_routes(const common_params & params, server_context & ctx_
queue_tasks(ctx_server.impl->queue_tasks),
queue_results(ctx_server.impl->queue_results) {
init_routes();
+
+ // note: this must be registered before load_model()
+ // so that on sleep phase, the callback is called before ctx is destroyed
+ queue_tasks.on_sleeping_state([this](bool is_sleeping) {
+ update_cached_responses(is_sleeping);
+ });
+}
+
+static json get_res_model_info(const server_context_meta & meta) {
+ // note: do NOT use ctx_server here, otherwise it's not possible to use this during sleep
+
+ return {
+ {"id", meta.model_name},
+ {"aliases", meta.model_aliases},
+ {"tags", meta.model_tags},
+ {"object", "model"},
+ {"created", std::time(0)},
+ {"owned_by", "llamacpp"},
+ {"meta", {
+ {"vocab_type", meta.model_vocab_type},
+ {"n_vocab", meta.model_vocab_n_tokens},
+ {"n_ctx", meta.slot_n_ctx},
+ {"n_ctx_train", meta.model_n_ctx_train},
+ {"n_embd", meta.model_n_embd_inp},
+ {"n_params", meta.model_n_params},
+ {"size", meta.model_size},
+ {"ftype", meta.model_ftype},
+ }},
+ };
+}
+
+static json get_res_models(const server_context_meta & meta) {
+ // note: do NOT use ctx_server here, otherwise it's not possible to use this during sleep
+
+ return {
+ {"models", {
+ {
+ {"name", meta.model_name},
+ {"model", meta.model_name},
+ {"modified_at", ""},
+ {"size", ""},
+ {"digest", ""}, // dummy value, llama.cpp does not support managing model file's hash
+ {"type", "model"},
+ {"description", ""},
+ {"tags", {""}},
+ {"capabilities", meta.has_mtmd ? json({"completion","multimodal"}) : json({"completion"})},
+ {"parameters", ""},
+ {"details", {
+ {"parent_model", ""},
+ {"format", "gguf"},
+ {"family", ""},
+ {"families", {""}},
+ {"parameter_size", ""},
+ {"quantization_level", ""}
+ }}
+ }
+ }},
+ {"object", "list"},
+ {"data", {
+ get_res_model_info(meta),
+ }}
+ };
+}
+
+static json get_res_props(const server_context_meta & meta, const common_params & params, bool is_sleeping) {
+ // note: do NOT use ctx_server here, otherwise it's not possible to use this during sleep
+
+ task_params tparams;
+ tparams.sampling = params.sampling;
+ json default_generation_settings_for_props = json {
+ { "params", tparams.to_json(true) },
+ { "n_ctx", meta.slot_n_ctx },
+ };
+
+ std::string tmpl_default = common_chat_templates_source(meta.chat_params.tmpls.get(), "");
+ std::string tmpl_tools = common_chat_templates_source(meta.chat_params.tmpls.get(), "tool_use");
+
+ json props = {
+ { "default_generation_settings", default_generation_settings_for_props },
+ { "total_slots", params.n_parallel },
+ { "model_alias", meta.model_name },
+ { "model_ftype", meta.model_ftype },
+ { "model_path", meta.model_path },
+ { "modalities", json {
+ {"vision", meta.has_inp_image},
+ {"video", meta.has_inp_video},
+ {"audio", meta.has_inp_audio},
+ } },
+ { "media_marker", get_media_marker() },
+ { "endpoint_slots", params.endpoint_slots },
+ { "endpoint_props", params.endpoint_props },
+ { "endpoint_metrics", params.endpoint_metrics },
+ { "ui", params.ui },
+ { "ui_settings", meta.json_ui_settings },
+ { "chat_template", tmpl_default },
+ { "chat_template_caps", meta.chat_template_caps },
+ { "bos_token", meta.bos_token_str },
+ { "eos_token", meta.eos_token_str },
+ { "build_info", meta.build_info },
+ { "is_sleeping", is_sleeping },
+ { "cors_proxy_enabled", params.ui_mcp_proxy },
+ };
+ if (params.use_jinja) {
+ if (!tmpl_tools.empty()) {
+ props["chat_template_tool_use"] = tmpl_tools;
+ }
+ }
+
+ return props;
+}
+
+json server_routes::get_model_info() const {
+ return get_res_model_info(*meta);
}
void server_routes::init_routes() {
@@ -4451,41 +4581,64 @@ void server_routes::init_routes() {
};
this->get_metrics = [this](const server_http_req & req) {
- auto res = create_response();
+ auto res = create_response(true);
if (!params.endpoint_metrics) {
res->error(format_error_response("This server does not support metrics endpoint. Start it with `--metrics`", ERROR_TYPE_NOT_SUPPORTED));
return res;
}
- // request slots data using task queue
- {
- server_task task(SERVER_TASK_TYPE_METRICS);
- task.id = res->rd.get_new_id();
+ // render response using cached_metrics
+ auto use_cached_metrics = [&]() {
+ std::unique_lock<std::mutex> lock(mutex_cache);
+ res->headers["Process-Start-Time-Unix"] = std::to_string(cached_metrics.t_start);
+ server_task_result_metrics tmp;
+ tmp.metrics = cached_metrics;
+ res->content_type = "text/plain; version=0.0.4";
+ res->status = 200;
+ res->data = tmp.to_metrics();
// the gauges are averaged over the window between two scrapes
- task.metrics_reset_bucket = true;
- res->rd.post_task(std::move(task), true); // high-priority task
- }
+ cached_metrics.reset_bucket();
+ should_reset_buckets = true;
+ };
- // get the result
- auto result = res->rd.next(req.should_stop);
- if (!result) {
- // connection was closed
- GGML_ASSERT(req.should_stop());
- return res;
- }
+ if (queue_tasks.is_sleeping()) {
+ use_cached_metrics();
- if (result->is_error()) {
- res->error(result->to_json());
- return res;
- }
+ } else {
+ // request slots data using task queue
+ {
+ server_task task(SERVER_TASK_TYPE_METRICS);
+ task.id = res->rd.get_new_id();
+ // the gauges are averaged over the window between two scrapes
+ task.metrics_reset_bucket = true;
+ res->rd.post_task(std::move(task), true); // high-priority task
+ }
- auto res_task = dynamic_cast<server_task_result_metrics*>(result.get());
- GGML_ASSERT(res_task != nullptr);
+ // a task posted right before sleeping is never processed, do not wait for it
+ auto result = res->rd.next([&]{
+ return req.should_stop() || queue_tasks.is_sleeping();
+ });
+ if (!result) {
+ if (!req.should_stop()) {
+ use_cached_metrics();
+ }
+ return res;
+ }
+
+ if (result->is_error()) {
+ res->error(result->to_json());
+ return res;
+ }
+
+ auto res_task = dynamic_cast<server_task_result_metrics*>(result.get());
+ GGML_ASSERT(res_task != nullptr);
+
+ res->headers["Process-Start-Time-Unix"] = std::to_string(res_task->metrics.t_start);
+ res->content_type = "text/plain; version=0.0.4";
+ res->status = 200;
+ res->data = res_task->to_metrics();
+ }
- res->headers["Process-Start-Time-Unix"] = std::to_string(res_task->metrics.t_start);
- res->content_type = "text/plain; version=0.0.4";
- res->status = 200;
- res->data = res_task->to_metrics();
return res;
};
@@ -4498,7 +4651,7 @@ void server_routes::init_routes() {
// request slots data using task queue
{
- server_task task(SERVER_TASK_TYPE_METRICS);
+ server_task task(SERVER_TASK_TYPE_SLOT_GET);
task.id = res->rd.get_new_id();
res->rd.post_task(std::move(task), true); // high-priority task
}
@@ -4516,7 +4669,7 @@ void server_routes::init_routes() {
return res;
}
- auto * res_task = dynamic_cast<server_task_result_metrics*>(result.get());
+ auto * res_task = dynamic_cast<server_task_result_slots*>(result.get());
GGML_ASSERT(res_task != nullptr);
// optionally return "fail_on_no_slot" error
@@ -4566,53 +4719,13 @@ void server_routes::init_routes() {
this->get_props = [this](const server_http_req &) {
auto res = create_response(true);
-
- // this endpoint can be accessed during sleeping
- // the next LOC is to avoid someone accidentally use ctx_server
- bool ctx_server; // do NOT delete this line
- GGML_UNUSED(ctx_server);
-
- task_params tparams;
- tparams.sampling = params.sampling;
- json default_generation_settings_for_props = json {
- { "params", tparams.to_json(true) },
- { "n_ctx", meta->slot_n_ctx },
- };
-
- std::string tmpl_default = common_chat_templates_source(meta->chat_params.tmpls.get(), "");
- std::string tmpl_tools = common_chat_templates_source(meta->chat_params.tmpls.get(), "tool_use");
-
- json props = {
- { "default_generation_settings", default_generation_settings_for_props },
- { "total_slots", params.n_parallel },
- { "model_alias", meta->model_name },
- { "model_ftype", meta->model_ftype },
- { "model_path", meta->model_path },
- { "modalities", json {
- {"vision", meta->has_inp_image},
- {"video", meta->has_inp_video},
- {"audio", meta->has_inp_audio},
- } },
- { "media_marker", get_media_marker() },
- { "endpoint_slots", params.endpoint_slots },
- { "endpoint_props", params.endpoint_props },
- { "endpoint_metrics", params.endpoint_metrics },
- { "ui", params.ui },
- { "ui_settings", meta->json_ui_settings },
- { "chat_template", tmpl_default },
- { "chat_template_caps", meta->chat_template_caps },
- { "bos_token", meta->bos_token_str },
- { "eos_token", meta->eos_token_str },
- { "build_info", meta->build_info },
- { "is_sleeping", queue_tasks.is_sleeping() },
- { "cors_proxy_enabled", params.ui_mcp_proxy },
- };
- if (params.use_jinja) {
- if (!tmpl_tools.empty()) {
- props["chat_template_tool_use"] = tmpl_tools;
- }
+ // note: do NOT use ctx_server here, this endpoint must be accessible during sleep
+ if (queue_tasks.is_sleeping()) {
+ std::unique_lock<std::mutex> lock(mutex_cache);
+ res->ok(cached_props);
+ } else {
+ res->ok(get_res_props(*meta, params, false));
}
- res->ok(props);
return res;
};
@@ -4874,42 +4987,13 @@ void server_routes::init_routes() {
this->get_models = [this](const server_http_req &) {
auto res = create_response(true);
-
- // this endpoint can be accessed during sleeping
- // the next LOC is to avoid someone accidentally use ctx_server
- bool ctx_server; // do NOT delete this line
- GGML_UNUSED(ctx_server);
-
- json models = {
- {"models", {
- {
- {"name", meta->model_name},
- {"model", meta->model_name},
- {"modified_at", ""},
- {"size", ""},
- {"digest", ""}, // dummy value, llama.cpp does not support managing model file's hash
- {"type", "model"},
- {"description", ""},
- {"tags", {""}},
- {"capabilities", meta->has_mtmd ? json({"completion","multimodal"}) : json({"completion"})},
- {"parameters", ""},
- {"details", {
- {"parent_model", ""},
- {"format", "gguf"},
- {"family", ""},
- {"families", {""}},
- {"parameter_size", ""},
- {"quantization_level", ""}
- }}
- }
- }},
- {"object", "list"},
- {"data", {
- get_model_info(),
- }}
- };
-
- res->ok(models);
+ // note: do NOT use ctx_server here, this endpoint must be accessible during sleep
+ if (queue_tasks.is_sleeping()) {
+ std::unique_lock<std::mutex> lock(mutex_cache);
+ res->ok(cached_models);
+ } else {
+ res->ok(get_res_models(*meta));
+ }
return res;
};
@@ -5119,27 +5203,6 @@ void server_routes::init_routes() {
};
}
-json server_routes::get_model_info() const {
- return json {
- {"id", meta->model_name},
- {"aliases", meta->model_aliases},
- {"tags", meta->model_tags},
- {"object", "model"},
- {"created", std::time(0)},
- {"owned_by", "llamacpp"},
- {"meta", {
- {"vocab_type", meta->model_vocab_type},
- {"n_vocab", meta->model_vocab_n_tokens},
- {"n_ctx", meta->slot_n_ctx},
- {"n_ctx_train", meta->model_n_ctx_train},
- {"n_embd", meta->model_n_embd_inp},
- {"n_params", meta->model_n_params},
- {"size", meta->model_size},
- {"ftype", meta->model_ftype},
- }},
- };
-}
-
std::unique_ptr<server_res_generator> server_routes::handle_slots_save(const server_http_req & req, int id_slot) {
auto res = create_response();
const json request_data = json::parse(req.body);
@@ -5388,3 +5451,24 @@ std::unique_ptr<server_res_generator> server_routes::handle_count_tokens(const l
res->ok(response);
return res;
}
+
+void server_routes::update_cached_responses(bool is_sleeping) {
+ // caller is task_queue, so ctx_server can be accessed without holding locks
+ std::unique_lock<std::mutex> lock(mutex_cache);
+
+ if (is_sleeping) {
+ cached_models = get_res_models(*meta);
+ cached_props = get_res_props(*meta, params, true);
+ cached_metrics = ctx_server.get_metrics();
+
+ should_reset_buckets = false;
+
+ SRV_DBG("%s\n", "cached responses updated");
+
+ } else if (should_reset_buckets) {
+ // a scrape during sleep already reported these buckets
+ ctx_server.reset_metrics_bucket();
+
+ should_reset_buckets = false;
+ }
+}
diff --git a/tools/server/server-context.h b/tools/server/server-context.h
index f9ab1132b1..764df0e085 100644
--- a/tools/server/server-context.h
+++ b/tools/server/server-context.h
@@ -8,6 +8,7 @@
#include <cstddef>
#include <memory>
+#include <mutex>
#include <set>
struct server_context_impl; // private implementation
@@ -174,9 +175,19 @@ private:
std::unique_ptr<const server_context_meta> meta;
const common_params & params;
- const server_context_impl & ctx_server;
+ server_context_impl & ctx_server;
server_queue & queue_tasks;
server_response & queue_results;
std::unique_ptr<server_res_generator> create_response(bool bypass_sleep = false);
+
+ // cached responses, to be used during sleep
+ std::mutex mutex_cache;
+ json cached_models = nullptr;
+ json cached_props = nullptr;
+ server_metrics cached_metrics;
+ // set when a scrape during sleep already reported the throughput buckets
+ bool should_reset_buckets = false;
+ // call right before sleep to update the cached responses
+ void update_cached_responses(bool is_sleeping);
};
diff --git a/tools/server/server-queue.cpp b/tools/server/server-queue.cpp
index 2bcc9bd8f2..78169e9a5d 100644
--- a/tools/server/server-queue.cpp
+++ b/tools/server/server-queue.cpp
@@ -3,6 +3,7 @@
#include "log.h"
+#include <algorithm>
#include <chrono>
#include <thread>
@@ -20,6 +21,10 @@
// server_queue
//
+static bool task_resets_idle_timer(server_task_type type) {
+ return type != SERVER_TASK_TYPE_METRICS;
+}
+
int server_queue::post(server_task && task, bool front) {
std::unique_lock<std::mutex> lock(mutex_tasks);
GGML_ASSERT(task.id != -1);
@@ -27,20 +32,24 @@ int server_queue::post(server_task && task, bool front) {
if (task.type == SERVER_TASK_TYPE_CANCEL) {
cleanup_pending_task(task.id_target);
}
- const int task_id = task.id;
+ const int task_id = task.id;
+ const bool reset_timer = task_resets_idle_timer(task.type);
QUE_DBG("new task, id = %d, front = %d\n", task_id, front);
if (front) {
queue_tasks.push_front(std::move(task));
} else {
queue_tasks.push_back(std::move(task));
}
- time_last_task = ggml_time_ms();
+ if (reset_timer) {
+ time_last_task = ggml_time_ms();
+ }
condition_tasks.notify_one();
return task_id;
}
int server_queue::post(std::vector<server_task> && tasks, bool front) {
std::unique_lock<std::mutex> lock(mutex_tasks);
+ bool reset_timer = false;
for (auto & task : tasks) {
if (task.id == -1) {
task.id = id++;
@@ -49,6 +58,7 @@ int server_queue::post(std::vector<server_task> && tasks, bool front) {
if (task.type == SERVER_TASK_TYPE_CANCEL) {
cleanup_pending_task(task.id_target);
}
+ reset_timer |= task_resets_idle_timer(task.type);
QUE_DBG("new task, id = %d/%d, front = %d\n", task.id, (int) tasks.size(), front);
if (front) {
queue_tasks.push_front(std::move(task));
@@ -56,7 +66,9 @@ int server_queue::post(std::vector<server_task> && tasks, bool front) {
queue_tasks.push_back(std::move(task));
}
}
- time_last_task = ggml_time_ms();
+ if (reset_timer) {
+ time_last_task = ggml_time_ms();
+ }
condition_tasks.notify_one();
return 0;
}
@@ -294,11 +306,14 @@ void server_queue::start_loop(int64_t idle_sleep_ms) {
QUE_DBG("%s", "update slots\n");
// this will run the main inference process for all slots
+ const int64_t t_update_slots = ggml_time_ms();
callback_update_slots();
{
// update_slots() may take a while to finish, we need to make sure it's not counted as idle
+ // shift instead of reset, so that non-task_resets_idle_timer tasks do not delay the sleep
std::unique_lock<std::mutex> lock(mutex_tasks);
- time_last_task = ggml_time_ms();
+ const int64_t now = ggml_time_ms();
+ time_last_task = std::min(now, time_last_task + (now - t_update_slots));
}
QUE_DBG("%s", "waiting for new tasks\n");
@@ -312,7 +327,10 @@ void server_queue::start_loop(int64_t idle_sleep_ms) {
if (should_sleep()) {
QUE_INF("%s", "entering sleeping state\n");
sleeping = true;
- callback_sleeping_state(true);
+ // Call order cb0 -> cb1 -> cb{N}
+ for (auto & cb : callback_sleeping_state) {
+ cb(true);
+ }
req_stop_sleeping = false;
// wait until we are requested to exit sleeping state
condition_tasks.wait(lock, [&]{
@@ -323,7 +341,10 @@ void server_queue::start_loop(int64_t idle_sleep_ms) {
}
QUE_INF("%s", "exiting sleeping state\n");
req_stop_sleeping = false;
- callback_sleeping_state(false);
+ // Call order cb{N} -> cb1 -> cb0
+ for (size_t i = callback_sleeping_state.size(); i > 0; i--) {
+ callback_sleeping_state[i - 1](false);
+ }
sleeping = false;
time_last_task = ggml_time_ms();
condition_tasks.notify_all(); // notify wait_until_no_sleep()
diff --git a/tools/server/server-queue.h b/tools/server/server-queue.h
index 52d30095c1..e17733a743 100644
--- a/tools/server/server-queue.h
+++ b/tools/server/server-queue.h
@@ -44,7 +44,7 @@ private:
// callback functions
std::function<bool(server_task &&, bool)> callback_new_task;
std::function<void(void)> callback_update_slots;
- std::function<void(bool)> callback_sleeping_state;
+ std::vector<std::function<void(bool)>> callback_sleeping_state;
public:
~server_queue() { worker_stop(); }
@@ -86,6 +86,7 @@ public:
*
* Sleeping procedure (disabled if idle_sleep_ms < 0):
* - If there is no task after idle_sleep_ms, enter sleeping state
+ * note: metrics tasks are processed as usual, but do not reset the idle timer
* - Call callback_sleeping_state(true)
* - Wait until req_stop_sleeping is set to true
* - Call callback_sleeping_state(false)
@@ -127,18 +128,12 @@ public:
}
// Register callback for sleeping state change; multiple callbacks are allowed
- // note: when entering sleeping state, the callback is called AFTER sleeping is set to true
- // when leaving sleeping state, the callback is called BEFORE sleeping is set to false
+ // for example: register order cb0, cb1, cb2
+ // entering sleep: queue.sleeping = true --> cb0(true) --> cb1(true) --> cb2(true)
+ // leaving sleep: cb2(false) --> cb1(false) --> cb0(false) --> queue.sleeping = false
+ // note: caller will hold mutex_tasks while calling the callbacks
void on_sleeping_state(std::function<void(bool)> callback) {
- if (callback_sleeping_state) {
- auto prev_callback = std::move(callback_sleeping_state);
- callback_sleeping_state = [prev_callback, callback](bool sleeping) {
- prev_callback(sleeping);
- callback(sleeping);
- };
- } else {
- callback_sleeping_state = std::move(callback);
- }
+ callback_sleeping_state.push_back(std::move(callback));
}
private:
diff --git a/tools/server/server-task.cpp b/tools/server/server-task.cpp
index 64afbc5edf..258cdcf8fb 100644
--- a/tools/server/server-task.cpp
+++ b/tools/server/server-task.cpp
@@ -1512,10 +1512,15 @@ json server_task_result_error::to_json() {
//
// server_task_result_metrics
//
-json server_task_result_metrics::to_json() {
+json server_task_result_slots::to_json() {
return slots_data;
}
+json server_task_result_metrics::to_json() {
+ // not used, /metrics renders prometheus text via to_metrics()
+ return json{};
+}
+
// metrics definition: https://prometheus.io/docs/practices/naming/#metric-names
std::string server_task_result_metrics::to_metrics() {
const std::vector<metric_item> counters = {
diff --git a/tools/server/server-task.h b/tools/server/server-task.h
index b6da4d4bd6..25ff015122 100644
--- a/tools/server/server-task.h
+++ b/tools/server/server-task.h
@@ -22,6 +22,7 @@ enum server_task_type {
SERVER_TASK_TYPE_CONTROL,
SERVER_TASK_TYPE_NEXT_RESPONSE,
SERVER_TASK_TYPE_METRICS,
+ SERVER_TASK_TYPE_SLOT_GET,
SERVER_TASK_TYPE_SLOT_SAVE,
SERVER_TASK_TYPE_SLOT_RESTORE,
SERVER_TASK_TYPE_SLOT_ERASE,
@@ -489,22 +490,16 @@ struct server_task_result_error : server_task_result {
virtual json to_json() override;
};
+// used by /metrics API
struct server_task_result_metrics : server_task_result {
// these are immediate stats, not accumulated (server_metrics is cumulative)
- int n_idle_slots;
- int n_processing_slots;
- int n_tasks_deferred;
+ int n_processing_slots = 0;
+ int n_tasks_deferred = 0;
server_metrics metrics;
- // while we can also use std::vector<server_slot> this requires copying the slot object which can be quite messy
- // therefore, we use json to temporarily store the slot.to_json() result
- json slots_data = json::array();
-
- // used by /slots API
virtual json to_json() override;
- // used by /metrics API
struct metric_item {
std::string name;
std::string description;
@@ -513,6 +508,17 @@ struct server_task_result_metrics : server_task_result {
std::string to_metrics();
};
+// used by /slots API
+struct server_task_result_slots : server_task_result {
+ int n_idle_slots = 0;
+
+ // while we can also use std::vector<server_slot> this requires copying the slot object which can be quite messy
+ // therefore, we use json to temporarily store the slot.to_json() result
+ json slots_data = json::array();
+
+ virtual json to_json() override;
+};
+
struct server_task_result_slot_save_load : server_task_result {
std::string filename;
bool is_save; // true = save, false = load
diff --git a/tools/server/tests/unit/test_sleep.py b/tools/server/tests/unit/test_sleep.py
index 3374165e83..515f7077d3 100644
--- a/tools/server/tests/unit/test_sleep.py
+++ b/tools/server/tests/unit/test_sleep.py
@@ -11,6 +11,35 @@ def create_server():
server = ServerPreset.tinyllama2()
+def is_sleeping(server: ServerProcess) -> bool:
+ res = server.make_request("GET", "/props")
+ assert res.status_code == 200
+ return res.body["is_sleeping"]
+
+
+def wait_for_sleep(server: ServerProcess, timeout: float = 10.0):
+ start = time.time()
+ while time.time() - start < timeout:
+ if is_sleeping(server):
+ return
+ time.sleep(0.1)
+ raise TimeoutError("server did not go to sleep")
+
+
+def fetch_metrics(server: ServerProcess) -> str:
+ res = server.make_request("GET", "/metrics")
+ assert res.status_code == 200
+ assert isinstance(res.body, str)
+ return res.body
+
+
+def get_metric(text: str, name: str) -> float:
+ prefix = f"llamacpp:{name} "
+ values = [ln for ln in text.splitlines() if ln.startswith(prefix)]
+ assert len(values) == 1, f"{name} not found in metrics"
+ return float(values[0][len(prefix):])
+
+
def test_server_sleep():
global server
server.sleep_idle_seconds = 1
@@ -25,6 +54,10 @@ def test_server_sleep():
res = server.make_request("GET", "/props")
assert res.status_code == 200
assert res.body["is_sleeping"] == True
+ res = server.make_request("GET", "/models")
+ assert res.status_code == 200
+ assert len(res.body["data"]) == 1
+ assert res.body["data"][0]["id"] == server.model_alias
# make a generation request to wake up the server
res = server.make_request("POST", "/completion", data={
@@ -37,3 +70,58 @@ def test_server_sleep():
res = server.make_request("GET", "/props")
assert res.status_code == 200
assert res.body["is_sleeping"] == False
+
+
+def test_server_sleep_read_only_endpoints():
+ global server
+ server.sleep_idle_seconds = 1
+ server.server_metrics = True
+ server.start()
+
+ res = server.make_request("POST", "/completion", data={
+ "n_predict": 4,
+ "prompt": "Hello",
+ })
+ assert res.status_code == 200
+
+ # the first scrape resets the throughput buckets, so that the second one reports
+ # the same zero rates as the snapshot taken on entering sleep
+ fetch_metrics(server)
+ metrics_awake = fetch_metrics(server)
+ assert get_metric(metrics_awake, "tokens_predicted_total") > 0
+
+ wait_for_sleep(server)
+
+ # during sleep, metrics are served from the snapshot taken right before sleeping
+ assert fetch_metrics(server) == metrics_awake
+
+ # scraping /metrics must not wake the server up
+ assert is_sleeping(server)
+
+
+def test_server_sleep_metrics_buckets():
+ global server
+ server.sleep_idle_seconds = 1
+ server.server_metrics = True
+ server.start()
+
+ res = server.make_request("POST", "/completion", data={
+ "n_predict": 8,
+ "prompt": "Hello",
+ })
+ assert res.status_code == 200
+
+ wait_for_sleep(server)
+
+ # the first scrape reports the throughput of the last generation
+ assert get_metric(fetch_metrics(server), "predicted_tokens_seconds") > 0
+
+ # nothing runs while sleeping, so the next scrapes report an empty window
+ assert get_metric(fetch_metrics(server), "predicted_tokens_seconds") == 0
+ assert is_sleeping(server)
+
+ # waking up must not report the buckets again
+ res = server.make_request("POST", "/tokenize", data={"content": "Hello"})
+ assert res.status_code == 200
+ assert is_sleeping(server) == False
+ assert get_metric(fetch_metrics(server), "predicted_tokens_seconds") == 0