Skip to content

LLM inference in C/C++

C++ ggml

server: fix router args not being forwarded to child instances (#24760)

Xuan-Son Nguyen968c43891abb
diff --git a/tools/server/server-models.cpp b/tools/server/server-models.cpp
index ff9a0df12f..cc7184165d 100644
--- a/tools/server/server-models.cpp
+++ b/tools/server/server-models.cpp
@@ -351,6 +351,12 @@ void server_models::load_models() {
source_map[name] = SERVER_MODEL_SOURCE_PRESET;
}
+ // overlay router's own CLI args on top of every model preset so that
+ // e.g. `llama-server --temp 0` is honoured by all child processes
+ for (auto & [name, preset] : final_presets) {
+ preset.merge(base_preset);
+ }
+
auto get_source = [&](const std::string & name) {
return source_map.count(name) ? source_map.at(name) : SERVER_MODEL_SOURCE_PRESET;
};