LLM inference in C/C++
C++ ggml
server: (router) lazy-load startup_models after main setup (#27424)
Xuan-Son Nguyen9855ad69d38c
diff --git a/tools/server/README.md b/tools/server/README.mdindex b63a0e6dac..f5d747eee8 100644--- a/tools/server/README.md+++ b/tools/server/README.md@@ -1757,7 +1757,7 @@ The precedence rule for preset options is as follows:3. **Global options** defined in the preset file (`[*]`)We also offer additional options that are exclusive to presets (these aren't treated as command-line arguments):-- `load-on-startup` (boolean): Controls whether the model loads automatically when the server starts+- `load-on-startup` (boolean): Controls whether the model loads automatically when the server starts. Only applies at startup: if the model list is reloaded later (for example after editing the preset file), a newly added model is listed but not loaded- `stop-timeout` (int, seconds): After requested unload, wait for this many seconds before forcing termination (default: 10)- `dedup-cache-models` (boolean): When the preset uses `hf-repo` pointing to a model that is already downloaded, hide the corresponding cached model entry from `GET /models` (the preset entry remains visible). Set it in the `[*]` section to apply to all presets.diff --git a/tools/server/server-models.cpp b/tools/server/server-models.cppindex 35b9355700..d605451941 100644--- a/tools/server/server-models.cpp+++ b/tools/server/server-models.cpp@@ -672,24 +672,26 @@ void server_models::load_models() {apply_hidden();log_available_models();- std::vector<std::string> models_to_load;- for (const auto & [name, inst] : mapping) {- std::string val;- if (inst.meta.preset.get_option(COMMON_ARG_PRESET_LOAD_ON_STARTUP, val) && common_arg_utils::is_truthy(val)) {- models_to_load.push_back(name);+ // skipped on reload, see startup_models+ if (startup_models.has_value()) {+ std::vector<std::string> models_to_load;+ for (const auto & [name, inst] : mapping) {+ std::string val;+ if (inst.meta.preset.get_option(COMMON_ARG_PRESET_LOAD_ON_STARTUP, val) && common_arg_utils::is_truthy(val)) {+ models_to_load.push_back(name);+ }}- }- if ((int)models_to_load.size() > base_params.models_max) {- throw std::runtime_error(string_format(- "number of models to load on startup (%zu) exceeds models_max (%d)",- models_to_load.size(), base_params.models_max));+ if ((int)models_to_load.size() > base_params.models_max) {+ throw std::runtime_error(string_format(+ "number of models to load on startup (%zu) exceeds models_max (%d)",+ models_to_load.size(), base_params.models_max));+ }++ // to be lazy-loaded after main() setup phase is completed+ startup_models = std::move(models_to_load);}lk.unlock();- for (const auto & name : models_to_load) {- SRV_INF("(startup) loading model %s\n", name.c_str());- load(name);- }} else {// RELOAD: diff the new preset list against the current mapping and reconcileis_reloading = true;@@ -819,8 +821,8 @@ void server_models::load_models() {inst.meta.update_caps();}- // add models that are new in this reload- std::vector<std::string> newly_added;+ // add models that are new in this reload, load-on-startup is not honored here since a+ // reload never spawns an instancefor (const auto & [name, preset] : final_presets) {if (mapping.find(name) == mapping.end()) {server_model_meta meta{@@ -841,42 +843,40 @@ void server_models::load_models() {// /* need_download */ false,};add_model(std::move(meta));- newly_added.push_back(name);}}apply_stop_timeout();apply_hidden();- // clear reload flag before unlocking for autoload - load() blocks on !is_reloading,- // so clearing it here (while still locked) prevents a deadlock in the autoload calls below+ // clear reload flag under the lock, this releases the load() calls waiting on !is_reloadingis_reloading = false;cv.notify_all();log_available_models();- // collect autoload candidates while still under the lock- std::vector<std::string> to_autoload;- for (const auto & name : newly_added) {- auto it = mapping.find(name);- if (it != mapping.end()) {- std::string val;- if (it->second.meta.preset.get_option(COMMON_ARG_PRESET_LOAD_ON_STARTUP, val) && common_arg_utils::is_truthy(val)) {- to_autoload.push_back(name);- }- }- }-lk.unlock();- for (const auto & name : to_autoload) {- SRV_INF("(reload) loading new model %s\n", name.c_str());- load(name);- }notify_sse("models_reload", "*");}}+void server_models::load_startup_models() {+ std::vector<std::string> to_load;+ {+ std::lock_guard<std::mutex> lk(mutex);+ if (!startup_models.has_value()) {+ return; // already drained+ }+ to_load = std::move(*startup_models);+ startup_models.reset();+ }+ for (const auto & name : to_load) {+ SRV_INF("(startup) loading model %s\n", name.c_str());+ load(name);+ }+}+void server_models::update_meta(const std::string & name, const server_model_meta & meta) {std::lock_guard<std::mutex> lk(mutex);auto it = mapping.find(name);diff --git a/tools/server/server-models.h b/tools/server/server-models.hindex 79b231cba8..5cbb6a801e 100644--- a/tools/server/server-models.h+++ b/tools/server/server-models.h@@ -136,6 +136,10 @@ private:// if true, the next get_meta() will trigger a reload of model listbool need_reload = false;+ // models marked with load-on-startup, unset once load_startup_models() drains it+ // no value means the startup phase is over, so a reload must not queue anything+ std::optional<std::vector<std::string>> startup_models{std::in_place};+// conv_id -> model name that currently serves its stream session, lets the resumable stream// routes go straight to the owning child instead of polling every one. populated when// proxy_request forwards a POST carrying an X-Conversation-Id. best effort: a stale entry just@@ -231,6 +235,9 @@ public:// - if a model is not running, it will be added or updated according to the sourcevoid load_models();+ // lazy-load startup_models, to be called after main() setup phase+ void load_startup_models();+// check if a model instance exists (thread-safe)bool has_model(const std::string & name);diff --git a/tools/server/server.cpp b/tools/server/server.cppindex 230b578e07..5fe2729ba1 100644--- a/tools/server/server.cpp+++ b/tools/server/server.cpp@@ -424,6 +424,18 @@ int llama_server(common_params & params, int argc, char ** argv) {ctx_http.stop();};+ try {+ models_routes->models.load_startup_models();+ } catch (const std::exception & e) {+ SRV_ERR("failed to load models on startup: %s\n", e.what());+ ctx_http.stop();+ if (ctx_http.thread.joinable()) {+ ctx_http.thread.join();+ }+ clean_up();+ return 1;+ }+} else {// setup clean up function, to be called before exitclean_up = [&ctx_http, &ctx_server, &mcp_mgr]() {