Skip to content

LLM inference in C/C++

C++ ggml

server: (router) lazy-load startup_models after main setup (#27424)

Xuan-Son Nguyen9855ad69d38c
diff --git a/tools/server/README.md b/tools/server/README.md
index b63a0e6dac..f5d747eee8 100644
--- a/tools/server/README.md
+++ b/tools/server/README.md
@@ -1757,7 +1757,7 @@ The precedence rule for preset options is as follows:
3. **Global options** defined in the preset file (`[*]`)
We also offer additional options that are exclusive to presets (these aren't treated as command-line arguments):
-- `load-on-startup` (boolean): Controls whether the model loads automatically when the server starts
+- `load-on-startup` (boolean): Controls whether the model loads automatically when the server starts. Only applies at startup: if the model list is reloaded later (for example after editing the preset file), a newly added model is listed but not loaded
- `stop-timeout` (int, seconds): After requested unload, wait for this many seconds before forcing termination (default: 10)
- `dedup-cache-models` (boolean): When the preset uses `hf-repo` pointing to a model that is already downloaded, hide the corresponding cached model entry from `GET /models` (the preset entry remains visible). Set it in the `[*]` section to apply to all presets.
diff --git a/tools/server/server-models.cpp b/tools/server/server-models.cpp
index 35b9355700..d605451941 100644
--- a/tools/server/server-models.cpp
+++ b/tools/server/server-models.cpp
@@ -672,24 +672,26 @@ void server_models::load_models() {
apply_hidden();
log_available_models();
- std::vector<std::string> models_to_load;
- for (const auto & [name, inst] : mapping) {
- std::string val;
- if (inst.meta.preset.get_option(COMMON_ARG_PRESET_LOAD_ON_STARTUP, val) && common_arg_utils::is_truthy(val)) {
- models_to_load.push_back(name);
+ // skipped on reload, see startup_models
+ if (startup_models.has_value()) {
+ std::vector<std::string> models_to_load;
+ for (const auto & [name, inst] : mapping) {
+ std::string val;
+ if (inst.meta.preset.get_option(COMMON_ARG_PRESET_LOAD_ON_STARTUP, val) && common_arg_utils::is_truthy(val)) {
+ models_to_load.push_back(name);
+ }
}
- }
- if ((int)models_to_load.size() > base_params.models_max) {
- throw std::runtime_error(string_format(
- "number of models to load on startup (%zu) exceeds models_max (%d)",
- models_to_load.size(), base_params.models_max));
+ if ((int)models_to_load.size() > base_params.models_max) {
+ throw std::runtime_error(string_format(
+ "number of models to load on startup (%zu) exceeds models_max (%d)",
+ models_to_load.size(), base_params.models_max));
+ }
+
+ // to be lazy-loaded after main() setup phase is completed
+ startup_models = std::move(models_to_load);
}
lk.unlock();
- for (const auto & name : models_to_load) {
- SRV_INF("(startup) loading model %s\n", name.c_str());
- load(name);
- }
} else {
// RELOAD: diff the new preset list against the current mapping and reconcile
is_reloading = true;
@@ -819,8 +821,8 @@ void server_models::load_models() {
inst.meta.update_caps();
}
- // add models that are new in this reload
- std::vector<std::string> newly_added;
+ // add models that are new in this reload, load-on-startup is not honored here since a
+ // reload never spawns an instance
for (const auto & [name, preset] : final_presets) {
if (mapping.find(name) == mapping.end()) {
server_model_meta meta{
@@ -841,42 +843,40 @@ void server_models::load_models() {
// /* need_download */ false,
};
add_model(std::move(meta));
- newly_added.push_back(name);
}
}
apply_stop_timeout();
apply_hidden();
- // clear reload flag before unlocking for autoload - load() blocks on !is_reloading,
- // so clearing it here (while still locked) prevents a deadlock in the autoload calls below
+ // clear reload flag under the lock, this releases the load() calls waiting on !is_reloading
is_reloading = false;
cv.notify_all();
log_available_models();
- // collect autoload candidates while still under the lock
- std::vector<std::string> to_autoload;
- for (const auto & name : newly_added) {
- auto it = mapping.find(name);
- if (it != mapping.end()) {
- std::string val;
- if (it->second.meta.preset.get_option(COMMON_ARG_PRESET_LOAD_ON_STARTUP, val) && common_arg_utils::is_truthy(val)) {
- to_autoload.push_back(name);
- }
- }
- }
-
lk.unlock();
- for (const auto & name : to_autoload) {
- SRV_INF("(reload) loading new model %s\n", name.c_str());
- load(name);
- }
notify_sse("models_reload", "*");
}
}
+void server_models::load_startup_models() {
+ std::vector<std::string> to_load;
+ {
+ std::lock_guard<std::mutex> lk(mutex);
+ if (!startup_models.has_value()) {
+ return; // already drained
+ }
+ to_load = std::move(*startup_models);
+ startup_models.reset();
+ }
+ for (const auto & name : to_load) {
+ SRV_INF("(startup) loading model %s\n", name.c_str());
+ load(name);
+ }
+}
+
void server_models::update_meta(const std::string & name, const server_model_meta & meta) {
std::lock_guard<std::mutex> lk(mutex);
auto it = mapping.find(name);
diff --git a/tools/server/server-models.h b/tools/server/server-models.h
index 79b231cba8..5cbb6a801e 100644
--- a/tools/server/server-models.h
+++ b/tools/server/server-models.h
@@ -136,6 +136,10 @@ private:
// if true, the next get_meta() will trigger a reload of model list
bool need_reload = false;
+ // models marked with load-on-startup, unset once load_startup_models() drains it
+ // no value means the startup phase is over, so a reload must not queue anything
+ std::optional<std::vector<std::string>> startup_models{std::in_place};
+
// conv_id -> model name that currently serves its stream session, lets the resumable stream
// routes go straight to the owning child instead of polling every one. populated when
// proxy_request forwards a POST carrying an X-Conversation-Id. best effort: a stale entry just
@@ -231,6 +235,9 @@ public:
// - if a model is not running, it will be added or updated according to the source
void load_models();
+ // lazy-load startup_models, to be called after main() setup phase
+ void load_startup_models();
+
// check if a model instance exists (thread-safe)
bool has_model(const std::string & name);
diff --git a/tools/server/server.cpp b/tools/server/server.cpp
index 230b578e07..5fe2729ba1 100644
--- a/tools/server/server.cpp
+++ b/tools/server/server.cpp
@@ -424,6 +424,18 @@ int llama_server(common_params & params, int argc, char ** argv) {
ctx_http.stop();
};
+ try {
+ models_routes->models.load_startup_models();
+ } catch (const std::exception & e) {
+ SRV_ERR("failed to load models on startup: %s\n", e.what());
+ ctx_http.stop();
+ if (ctx_http.thread.joinable()) {
+ ctx_http.thread.join();
+ }
+ clean_up();
+ return 1;
+ }
+
} else {
// setup clean up function, to be called before exit
clean_up = [&ctx_http, &ctx_server, &mcp_mgr]() {