LLM inference in C/C++
C++ ggml
server: fix router eviction races with the existing queue (#29217)
Pascal991991118571
diff --git a/tools/server/server-models.cpp b/tools/server/server-models.cppindex 181d689318..d661d99841 100644--- a/tools/server/server-models.cpp+++ b/tools/server/server-models.cpp@@ -297,7 +297,7 @@ struct server_lru_sched {return;}queue.push_back({ model_id, 1, false });- SRV_INF("models_max reached, request for name=%s queued at position %zu\n",+ SRV_INF("request for name=%s queued at position %zu\n",model_id.c_str(), queue.size());}@@ -1223,15 +1223,16 @@ void server_models::request_stop(const std::string & name, bool send_exit) {void server_models::on_child_exit(const std::string & name, const std::shared_ptr<server_subproc> & proc, server_child_mode mode, int exit_code) {{std::lock_guard<std::mutex> lk(mutex);- stopping_models.erase(name);auto it = mapping.find(name);if (it == mapping.end() || it->second.subproc != proc) {+ stopping_models.erase(name);return; // entry erased, or a newer instance took the name}}if (mode == SERVER_CHILD_MODE_DOWNLOAD) {// instance will be cleaned up on next load_models() callstd::lock_guard<std::mutex> lk(mutex);+ stopping_models.erase(name);cv.notify_all();} else {update_status(name, {@@ -1301,6 +1302,9 @@ void server_models::update_status(const std::string & name, const update_status_auto & meta = it->second.meta;meta.status = args.status;meta.exit_code = args.exit_code;+ if (args.status == SERVER_MODEL_STATUS_UNLOADED) {+ stopping_models.erase(name);+ }if (!args.loaded_info.is_null()) {meta.loaded_info = args.loaded_info;}@@ -1440,10 +1444,15 @@ bool server_models::ensure_model_ready(const std::string & name, const std::funcif (!meta.has_value()) {throw std::runtime_error("model name=" + name + " is not found");}- if (meta->is_ready()) {+ bool stopping;+ {+ std::lock_guard<std::mutex> lk(mutex);+ stopping = stopping_models.count(name) > 0;+ }+ if (!stopping && meta->is_ready()) {return false; // ready for taking requests}- if (meta->status == SERVER_MODEL_STATUS_SLEEPING) {+ if (!stopping && meta->status == SERVER_MODEL_STATUS_SLEEPING) {return false; // child is sleeping but still running; new request will wake it up}@@ -1453,17 +1462,10 @@ bool server_models::ensure_model_ready(const std::string & name, const std::funcstd::unique_lock<std::mutex> lk(mutex);auto it = mapping.find(name);if (it != mapping.end() && it->second.meta.status == SERVER_MODEL_STATUS_UNLOADED) {- if (sched->has_capacity(lk) && sched->queue_empty(lk)) {- lk.unlock();- SRV_INF("model name=%s is not loaded, loading...\n", name.c_str());- load(name);- did_load = true;- } else {- // also queue when a slot looks free but others wait already, else they starve- sched->join(lk, name);- sched->tick(lk);- queued = true;- }+ // the queue entry protects the model from eviction until its waiters leave+ sched->join(lk, name);+ sched->tick(lk);+ queued = true;}}@@ -1484,6 +1486,19 @@ bool server_models::ensure_model_ready(const std::string & name, const std::funcif (it == mapping.end()) {break; // removed by another code path, nothing to wait for}+ if (stopping_models.count(name)) {+ // a stopping instance takes no new request, the next instance serves it+ if (!queued) {+ sched->join(lk, name);+ sched->tick(lk);+ queued = true;+ }+ if (should_stop && should_stop()) {+ throw std::runtime_error("request cancelled while waiting for model name=" + name);+ }+ cv.wait_for(lk, std::chrono::milliseconds(200));+ continue;+ }const server_model_status status = it->second.meta.status;if (status == SERVER_MODEL_STATUS_LOADED || status == SERVER_MODEL_STATUS_SLEEPING) {