LLM inference in C/C++
C++ ggml
server: (router) move model downloading to dedicated process (#24834)
Xuan-Son Nguyen721354fbdfb7
diff --git a/common/arg.cpp b/common/arg.cppindex 8f54b5c814..5297d90753 100644--- a/common/arg.cpp+++ b/common/arg.cpp@@ -396,7 +396,7 @@ static bool parse_bool_value(const std::string & value) {// CLI argument parsing functions//-bool common_params_handle_models(common_params & params, llama_example curr_ex) {+bool common_params_handle_models(common_params & params, llama_example curr_ex, common_download_callback * callback) {const bool spec_type_draft_mtp = std::find(params.speculative.types.begin(),params.speculative.types.end(),COMMON_SPECULATIVE_TYPE_DRAFT_MTP) != params.speculative.types.end();@@ -408,6 +408,10 @@ bool common_params_handle_models(common_params & params, llama_example curr_ex)opts.download_mtp = spec_type_draft_mtp;opts.download_mmproj = !params.no_mmproj && params.mmproj.path.empty() && params.mmproj.url.empty();+ if (callback) {+ opts.callback = callback;+ }+// sub-models (draft, mmproj, vocoder) are explicitly specified by the user,// so we should not auto-discover mtp/mmproj siblings for themcommon_download_opts sub_opts = opts;@@ -584,8 +588,11 @@ static bool common_params_parse_ex(int argc, char ** argv, common_params_contextthrow std::invalid_argument("error: --prompt-cache-all not supported in interactive mode yet\n");}- // export_graph_ops loads only metadata- const bool skip_model_download = ctx_arg.ex == LLAMA_EXAMPLE_EXPORT_GRAPH_OPS;+ const bool skip_model_download =+ // server will call common_params_handle_models() later, so we skip it here+ ctx_arg.ex == LLAMA_EXAMPLE_SERVER ||+ // export_graph_ops loads only metadata+ ctx_arg.ex == LLAMA_EXAMPLE_EXPORT_GRAPH_OPS;if (!skip_model_download) {// handle model and download@@ -594,7 +601,6 @@ static bool common_params_parse_ex(int argc, char ** argv, common_params_context// model is required (except for server)// TODO @ngxson : maybe show a list of available models in CLI in this caseif (params.model.path.empty()- && ctx_arg.ex != LLAMA_EXAMPLE_SERVER&& !params.usage&& !params.completion) {throw std::invalid_argument("error: --model is required\n");diff --git a/common/arg.h b/common/arg.hindex 0010f2a9ac..c061fc60f7 100644--- a/common/arg.h+++ b/common/arg.h@@ -1,6 +1,7 @@#pragma once#include "common.h"+#include "download.h"#include <set>#include <map>@@ -133,7 +134,10 @@ void common_params_add_preset_options(std::vector<common_arg> & args);// return true if the model is ready to use// throw an exception if there is an error that prevents the model from being used (e.g. network error, model not found, etc)// if params.skip_download is true, no downloads will be attempted. return false if the model is invalid or missing (e.g. ETag check failed)-bool common_params_handle_models(common_params & params, llama_example curr_ex);+bool common_params_handle_models(+ common_params & params,+ llama_example curr_ex,+ common_download_callback * callback = nullptr);// initialize argument parser context - used by test-arg-parser and presetcommon_params_context common_params_parser_init(common_params & params, llama_example ex, void(*print_usage)(int, char **) = nullptr);diff --git a/tools/server/README-dev.md b/tools/server/README-dev.mdindex 2796d28350..5959745e47 100644--- a/tools/server/README-dev.md+++ b/tools/server/README-dev.md@@ -204,9 +204,9 @@ Instead of building everything from the ground up (like what most AI agents willThe flow for downloading a new model:- POST request comes in --> `post_router_models` --> validation-- `server_models::download()` is called- - Sets up a new thread `inst.th` and runs the download inside-- If a stop request comes in, set `stop_download` to `true`+- A new `llama-server` subprocess will be spawned with special `SERVER_CHILD_MODE_DOWNLOAD`+- Child process runs the download and report status back to router via stdin/out+- If a stop request comes in, the router asks the child process to stop (same mechanism as running a model in child process)- Otherwise, upon completion, we call `load_models()` to refresh the list of models### Notable Related PRsdiff --git a/tools/server/server-context.cpp b/tools/server/server-context.cppindex 3f9391cacb..0a25b414ed 100644--- a/tools/server/server-context.cpp+++ b/tools/server/server-context.cpp@@ -931,6 +931,8 @@ private:bool sleeping = false;+ int64_t t_last_load_progress_ms = 0;+void destroy() {spec.reset();ctx_dft.reset();@@ -1244,6 +1246,10 @@ private:}if (has_mmproj) {+ if (callback_state) {+ callback_state(SERVER_STATE_LOADING, {{"stage", "mmproj_model"}});+ }+if (!is_resume) {mtmd_helper_log_set(common_log_default_callback, nullptr);}diff --git a/tools/server/server-context.h b/tools/server/server-context.hindex c7218a12ed..952f825f72 100644--- a/tools/server/server-context.h+++ b/tools/server/server-context.h@@ -53,7 +53,7 @@ struct server_context_meta {};enum server_state {- // SERVER_STATE_DOWNLOADING,+ SERVER_STATE_DOWNLOADING,SERVER_STATE_LOADING,SERVER_STATE_READY,SERVER_STATE_SLEEPING,@@ -61,6 +61,7 @@ enum server_state {static std::string server_state_to_str(server_state state) {switch (state) {+ case SERVER_STATE_DOWNLOADING: return "downloading";case SERVER_STATE_LOADING: return "loading";case SERVER_STATE_READY: return "ready";case SERVER_STATE_SLEEPING: return "sleeping";@@ -69,6 +70,7 @@ static std::string server_state_to_str(server_state state) {}static server_state server_state_from_str(const std::string & str) {+ if (str == "downloading") return SERVER_STATE_DOWNLOADING;if (str == "loading") return SERVER_STATE_LOADING;if (str == "ready") return SERVER_STATE_READY;if (str == "sleeping") return SERVER_STATE_SLEEPING;diff --git a/tools/server/server-models.cpp b/tools/server/server-models.cppindex 68eefdffac..a87e4e423e 100644--- a/tools/server/server-models.cpp+++ b/tools/server/server-models.cpp@@ -64,6 +64,17 @@ struct server_subproc {return sproc.has_value() && subprocess_alive(&sproc.value());}+ void request_exit() {+ if (sproc.has_value()) {+ FILE * stdin_file = subprocess_stdin(&sproc.value());+ if (stdin_file) {+ fprintf(stdin_file, "%s\n", CMD_ROUTER_TO_CHILD_EXIT);+ fflush(stdin_file);+ }+ }+ stopped.store(true, std::memory_order_relaxed);+ }+void terminate() {if (!sproc.has_value()) {return;@@ -323,7 +334,7 @@ void server_models::notify_sse(const std::string & event, const std::string & mo}void server_models::load_models() {- // Phase 1: load presets from all sources — pure I/O, no lock needed+ // Phase 1: load presets from all sources - pure I/O, no lock needed// 1. cached modelscommon_presets cached_models = ctx_preset.load_from_cache();SRV_INF("Loaded %zu cached model presets\n", cached_models.size());@@ -376,7 +387,7 @@ void server_models::load_models() {return source_map.count(name) ? source_map.at(name) : SERVER_MODEL_SOURCE_PRESET;};- // Helpers that read `mapping` — must be called while holding the lock.+ // Helpers that read `mapping` - must be called while holding the lock.std::unordered_set<std::string> custom_names;for (const auto & [name, preset] : custom_presets) custom_names.insert(name);auto join_set = [](const std::set<std::string> & s) {@@ -523,7 +534,7 @@ void server_models::load_models() {}}- // join outside the lock — monitoring thread calls update_status (needs lock)+ // join outside the lock - monitoring thread calls update_status (needs lock)lk.unlock();for (auto & th : threads_to_join) th.join();lk.lock();@@ -622,7 +633,7 @@ void server_models::load_models() {apply_stop_timeout();- // clear reload flag before unlocking for autoload — load() blocks on !is_reloading,+ // clear reload flag before unlocking for autoload - load() blocks on !is_reloading,// so clearing it here (while still locked) prevents a deadlock in the autoload calls belowis_reloading = false;cv.notify_all();@@ -815,17 +826,23 @@ void server_models::unload_lru() {}void server_models::load(const std::string & name) {- if (!has_model(name)) {- throw std::runtime_error("model name=" + name + " is not found");+ load(name, load_options{});+}++void server_models::load(const std::string & name, const load_options & opts) {+ if (!opts.custom_meta.has_value()) {+ if (!has_model(name)) {+ throw std::runtime_error("model name=" + name + " is not found");+ }+ unload_lru();}- unload_lru();std::unique_lock<std::mutex> lk(mutex);// edge case: block until any in-progress reload has finished so we always load// against the freshest preset and a consistent mapping statecv.wait(lk, [this]() { return !is_reloading; });- auto meta = mapping[name].meta;+ auto meta = opts.custom_meta.has_value() ? *opts.custom_meta : mapping[name].meta;if (meta.status != SERVER_MODEL_STATUS_UNLOADED) {SRV_INF("model %s is not ready\n", name.c_str());return;@@ -869,6 +886,12 @@ void server_models::load(const std::string & name) {std::vector<std::string> child_env = base_env; // copychild_env.push_back("LLAMA_SERVER_ROUTER_PORT=" + std::to_string(base_params.port));+ if (opts.mode == SERVER_CHILD_MODE_DOWNLOAD) {+ inst.meta.status = SERVER_MODEL_STATUS_DOWNLOADING;+ child_env.push_back("LLAMA_SERVER_CHILD_MODE=download");+ child_env.push_back("LLAMA_ARG_HF_REPO=" + name);+ }+SRV_INF("%s", "spawning server instance with args:\n");for (const auto & arg : child_args) {SRV_INF(" %s\n", arg.c_str());@@ -886,13 +909,17 @@ void server_models::load(const std::string & name) {if (result != 0) {throw std::runtime_error("failed to spawn server instance");}-- inst.stdin_file = subprocess_stdin(&inst.subproc->get());}// start a thread to manage the child process// captured variables are guaranteed to be destroyed only after the thread is joined- inst.th = std::thread([this, name, child_proc = inst.subproc, port = inst.meta.port, stop_timeout = inst.meta.stop_timeout]() {+ inst.th = std::thread([+ this, name,+ child_proc = inst.subproc,+ port = inst.meta.port,+ stop_timeout = inst.meta.stop_timeout,+ child_mode = opts.mode+ ]() {FILE * stdin_file = subprocess_stdin(&child_proc->get());FILE * stdout_file = subprocess_stdout(&child_proc->get()); // combined stdout/stderr@@ -925,7 +952,7 @@ void server_models::load(const std::string & name) {return is_stopping() || child_proc->stopped.load(std::memory_order_acquire);});}- // child crashed or finished on its own — skip graceful shutdown sequence+ // child crashed or finished on its own, skip graceful shutdown sequenceif (child_proc->stopped.load(std::memory_order_acquire)) {return;}@@ -973,10 +1000,14 @@ void server_models::load(const std::string & name) {subprocess_destroy(&child_proc->get());// update status and exit code- this->update_status(name, {- SERVER_MODEL_STATUS_UNLOADED,- exit_code- });+ if (child_mode == SERVER_CHILD_MODE_DOWNLOAD) {+ // instance will be cleaned up on next load_models() call+ } else {+ this->update_status(name, {+ SERVER_MODEL_STATUS_UNLOADED,+ exit_code+ });+ }SRV_INF("instance name=%s exited with status %d\n", name.c_str(), exit_code);});@@ -984,7 +1015,7 @@ void server_models::load(const std::string & name) {{auto & old_instance = mapping[name];// old process should have exited already, but just in case, we clean it up here- if (old_instance.subproc->is_alive()) {+ if (old_instance.subproc && old_instance.subproc->is_alive()) {SRV_WRN("old process for model name=%s is still alive, this is unexpected\n", name.c_str());old_instance.subproc->terminate(); // force kill}@@ -1001,92 +1032,13 @@ void server_models::load(const std::string & name) {cv.notify_all();}-// callback for model downloading functionality-struct server_models_download_res : public common_download_callback {- common_params_model model;- common_download_opts opts;-- std::function<bool()> should_stop;- std::function<void(const common_download_progress & p)> on_progress;-- bool is_ok = false;-- bool run() {- try {- common_download_model(model, opts);- is_ok = true;- } catch (const std::exception & e) {- auto model_name = model.get_name();- SRV_ERR("download failed for model name=%s: %s\n", model_name.c_str(), e.what());- is_ok = false;- }- return is_ok;- }- void on_start(const common_download_progress & p) override {- on_progress(p);- }- void on_update(const common_download_progress & p) override {- on_progress(p);- }- void on_done(const common_download_progress &, bool ok) override {- is_ok = ok;- }- bool is_cancelled() const override {- return should_stop();- }-};--void server_models::download(common_params_model && model, common_download_opts && opts) {- std::string name = model.get_name();- GGML_ASSERT(name == model.hf_repo);-- std::unique_lock<std::mutex> lk(mutex);- if (mapping.find(name) != mapping.end()) {- throw std::runtime_error("model name=" + name + " already exists");- }-- instance_t inst;- inst.meta.name = name;- inst.meta.status = SERVER_MODEL_STATUS_DOWNLOADING;- inst.subproc = std::make_shared<server_subproc>();-- auto dl = std::make_unique<server_models_download_res>();- dl->model = model; // copy- dl->opts = opts; // copy-- dl->should_stop = [sp = inst.subproc]() {- return sp->stopped.load(std::memory_order_relaxed);- };-- dl->on_progress = [this, name](const common_download_progress & p) {- update_download_progress(name, p, false);- };-- inst.th = std::thread([this, dl = std::move(dl)]() {- dl->opts.callback = dl.get();- bool ok = dl->run();- auto model_name = dl->model.get_name();- SRV_INF("download finished for model name=%s with status=%s\n",- model_name.c_str(), ok ? "success" : "failure");- update_download_progress(model_name, {}, true, ok);- // need_reload is set inside update_download_progress under the mutex;- // the next load_models() call will clean up this instance- });-- mapping[name] = std::move(inst);- notify_sse("status_update", name, {- {"status", server_model_status_to_string(SERVER_MODEL_STATUS_DOWNLOADING)},- });- cv.notify_all();-}-void server_models::unload(const std::string & name) {std::unique_lock<std::mutex> lk(mutex);auto it = mapping.find(name);if (it != mapping.end()) {if (it->second.meta.status == SERVER_MODEL_STATUS_DOWNLOADING) {SRV_INF("cancelling download for model name=%s\n", name.c_str());- it->second.subproc->stopped.store(true, std::memory_order_relaxed);+ it->second.subproc->request_exit();// for convenience, we wait the status change herewait(lk, name, [](const server_model_meta & new_meta) {return new_meta.status != SERVER_MODEL_STATUS_DOWNLOADING;@@ -1198,37 +1150,65 @@ void server_models::update_download_progress(const std::string & name, const com}bool server_models::remove(const std::string & name) {- auto meta = get_meta(name);+ // do everything under one lock acquisition; avoid get_meta() /+ // unload() because they can trigger load_models() which erases+ // transient DOWNLOADING / DOWNLOADED entries as a side-effect+ std::unique_lock<std::mutex> lk(mutex);- if (!meta.has_value()) {+ auto it = mapping.find(name);+ if (it == mapping.end()) {throw std::runtime_error("model name=" + name + " is not found");}- if (meta->source != SERVER_MODEL_SOURCE_CACHE) {+ if (it->second.meta.source != SERVER_MODEL_SOURCE_CACHE) {throw std::runtime_error("model name=" + name + " is not removable (not from cache)");}- unload(name); // cancel download or stop running instance- {- std::unique_lock<std::mutex> lk(mutex);- // a cancelled download lands on DOWNLOADED; a stopped instance lands on UNLOADED- wait(lk, name, [](const server_model_meta & new_meta) {- return new_meta.status == SERVER_MODEL_STATUS_UNLOADED- || new_meta.status == SERVER_MODEL_STATUS_DOWNLOADED;- });- // join before erasing - after status reaches UNLOADED/DOWNLOADED the thread no- // longer acquires this mutex, so joining while holding it is safe- if (mapping[name].th.joinable()) {- mapping[name].th.join();+ if (it->second.meta.status == SERVER_MODEL_STATUS_DOWNLOADING) {+ // cancel in-flight download+ SRV_INF("cancelling download for model name=%s\n", name.c_str());+ it->second.subproc->request_exit();+ } else if (it->second.meta.is_running()) {+ // stop running instance+ SRV_INF("stopping model instance name=%s\n", name.c_str());+ stopping_models.insert(name);+ if (it->second.meta.status == SERVER_MODEL_STATUS_LOADING) {+ it->second.subproc->terminate();}- // remove the model from disk (hold lock to prevent concurrent load)+ cv_stop.notify_all();+ }++ // wait until the monitoring thread finishes+ wait(lk, name, [](const server_model_meta & meta) {+ return meta.status == SERVER_MODEL_STATUS_UNLOADED+ || meta.status == SERVER_MODEL_STATUS_DOWNLOADED;+ });++ // re-find after wait - load_models() may have erased the entry during the wait+ it = mapping.find(name);+ if (it == mapping.end()) {+ // load_models() already joined the thread and erased the entry;+ // we just need to clean up the cached files on disk+ lk.unlock();bool ok = common_download_remove(name);- if (ok) {- mapping.erase(name);- }- SRV_INF("removing model name=%s from cache (%s)\n", name.c_str(), ok ? "succeeded" : "failed");+ SRV_INF("removing model name=%s from cache (%s)\n", name.c_str(), ok ? "succeeded" : "partial");notify_sse("model_remove", name, {});- return ok;+ return true;+ }++ // join before erasing - thread no longer acquires this mutex+ if (it->second.th.joinable()) {+ it->second.th.join();+ }++ // remove from disk (best-effort: cancelled downloads may have no cached files)+ bool ok = common_download_remove(name);+ mapping.erase(name);+ if (!ok) {+ SRV_WRN("removing model name=%s from disk returned false (no cached files?)\n", name.c_str());}+ SRV_INF("removing model name=%s from cache (%s)\n", name.c_str(), ok ? "succeeded" : "partial");+ notify_sse("model_remove", name, {});+ return true;}void server_models::wait(const std::string & name, std::function<bool(const server_model_meta &)> predicate) {@@ -1243,7 +1223,9 @@ void server_models::wait(std::unique_lock<std::mutex> & lk, const std::string &return predicate(it->second.meta);}- return false;+ // model was removed from mapping by another code path (e.g. load_models()).+ // nothing left to wait for - tell the caller to proceed.+ return true;});}@@ -1328,6 +1310,31 @@ void server_models::handle_child_state(const std::string & name, const std::stri}switch (state) {+ case SERVER_STATE_DOWNLOADING:+ {+ std::string result = json_value(payload, "result", std::string());+ std::string url = json_value(payload, "url", std::string());+ auto request_exit = [&]() {+ std::lock_guard<std::mutex> lk(mutex);+ auto it = mapping.find(name);+ if (it != mapping.end()) {+ return it->second.subproc->request_exit();+ }+ };+ if (result == "download_finished") {+ update_download_progress(name, {}, true, true);+ request_exit();+ } else if (result == "download_failed") {+ update_download_progress(name, {}, true, false);+ request_exit();+ } else if (!url.empty()) {+ common_download_progress p;+ p.url = url;+ p.downloaded = json_value(payload, "downloaded", (size_t)0);+ p.total = json_value(payload, "total", (size_t)0);+ update_download_progress(name, p, false);+ }+ } break;case SERVER_STATE_LOADING:{update_status(name, {@@ -1366,6 +1373,90 @@ bool server_child::is_child() {return router_port != nullptr;}+server_child_mode server_child::get_mode() {+ const char * mode = std::getenv("LLAMA_SERVER_CHILD_MODE");+ std::string mode_str(mode ? mode : "");+ if (mode_str == "download") {+ return SERVER_CHILD_MODE_DOWNLOAD;+ } else {+ return SERVER_CHILD_MODE_NORMAL;+ }+}++struct server_download_state : public common_download_callback {+ server_child * self;+ std::function<bool()> should_stop;+ std::atomic<int64_t> last_progress_time{0}; // multiple files downloading in different threads+ bool is_ok = false;++ server_download_state(server_child * s) : self(s) {}++ bool run(common_params & params) {+ try {+ common_params_handle_models(params, LLAMA_EXAMPLE_SERVER, this);+ is_ok = true;+ } catch (const std::exception & e) {+ auto model_name = params.model.get_name();+ SRV_ERR("download failed for model name=%s: %s\n", model_name.c_str(), e.what());+ is_ok = false;+ }+ return is_ok;+ }+ void on_progress(const common_download_progress & p) {+ json data = {+ {"url", p.url},+ {"downloaded", p.downloaded},+ {"total", p.total},+ };+ self->notify_to_router(server_state_to_str(SERVER_STATE_DOWNLOADING), data);+ }+ void on_start(const common_download_progress & p) override {+ on_progress(p);+ }+ void on_update(const common_download_progress & p) override {+ int64_t now = ggml_time_ms();+ // throttle progress updates to avoid flooding logs+ if (now - last_progress_time.load(std::memory_order_relaxed) >= 100) {+ on_progress(p);+ last_progress_time.store(now, std::memory_order_relaxed);+ }+ }+ void on_done(const common_download_progress & p, bool) override {+ on_progress(p);+ }+ bool is_cancelled() const override {+ return should_stop ? should_stop() : false;+ }+};++int server_child::run_download(common_params & params) {+ auto cancelled = std::make_shared<std::atomic<bool>>(false);++ // monitor stdin for cancellation command from the router+ std::thread signal_thread = setup([cancelled](int) {+ cancelled->store(true, std::memory_order_relaxed);+ });++ server_download_state dl(this);+ dl.should_stop = [cancelled]() {+ return cancelled->load(std::memory_order_relaxed);+ };++ bool ok = dl.run(params);++ notify_to_router(server_state_to_str(SERVER_STATE_DOWNLOADING), {+ {"result", ok ? "download_finished" : "download_failed"},+ });++ // router should send CMD_ROUTER_TO_CHILD_EXIT after receiving the result+ if (signal_thread.joinable()) {+ signal_thread.join();+ }++ SRV_INF("download completed %s\n", ok ? "successfully" : "with errors");+ return 0;+}+std::thread server_child::setup(const std::function<void(int)> & shutdown_handler) {// setup thread for monitoring stdinreturn std::thread([shutdown_handler]() {@@ -1639,7 +1730,7 @@ void server_models_routes::init_routes() {res_err(res, format_error_response("model is not found", ERROR_TYPE_INVALID_REQUEST));return res;}- if (!model->is_running()) {+ if (!model->is_running() && model->status != SERVER_MODEL_STATUS_DOWNLOADING) {res_err(res, format_error_response("model is not running", ERROR_TYPE_INVALID_REQUEST));return res;}@@ -1680,8 +1771,9 @@ void server_models_routes::init_routes() {model.hf_repo = name;opts.bearer_token = params.hf_token;- opts.download_mmproj = true;- opts.download_mtp = true;+ // note: we only check main model, no need sidecar here+ opts.download_mmproj = false;+ opts.download_mtp = false;// first, only check if the model is valid and can be downloadedopts.skip_download = true;@@ -1702,10 +1794,21 @@ void server_models_routes::init_routes() {throw std::invalid_argument("model validation failed, unable to download");}+ // reject if model already exists+ if (models.has_model(name)) {+ throw std::invalid_argument("model '" + name + "' already exists");+ }+// then, proceed with the actual download- opts.skip_download = false;SRV_INF("starting download for model '%s'\n", name.c_str());- models.download(std::move(model), std::move(opts));+ {+ server_models::load_options load_opts;+ load_opts.mode = SERVER_CHILD_MODE_DOWNLOAD;+ load_opts.custom_meta = server_model_meta{};+ load_opts.custom_meta->source = SERVER_MODEL_SOURCE_CACHE;+ load_opts.custom_meta->name = name;+ models.load(name, load_opts);+ }res_ok(res, {{"success", true}});return res;@@ -1719,10 +1822,7 @@ void server_models_routes::init_routes() {throw std::invalid_argument("model must be a non-empty string");}- bool ok = models.remove(name);- if (!ok) {- throw std::runtime_error("failed to remove model '" + name + "'");- }+ models.remove(name); // throws on errorres_ok(res, {{"success", true}});return res;diff --git a/tools/server/server-models.h b/tools/server/server-models.hindex 17759b00a5..9ed4aeead0 100644--- a/tools/server/server-models.h+++ b/tools/server/server-models.h@@ -40,6 +40,11 @@ enum server_model_source {SERVER_MODEL_SOURCE_CACHE,};+enum server_child_mode {+ SERVER_CHILD_MODE_NORMAL, // load the model and run normally+ SERVER_CHILD_MODE_DOWNLOAD, // download the model and exit+};+static std::string server_model_status_to_string(server_model_status status) {switch (status) {case SERVER_MODEL_STATUS_DOWNLOADING: return "downloading";@@ -105,7 +110,6 @@ private:std::shared_ptr<server_subproc> subproc; // shared between main thread and monitoring threadstd::thread th;server_model_meta meta;- FILE * stdin_file = nullptr;};std::mutex mutex;@@ -161,16 +165,19 @@ public:// return a copy of all model metadata (thread-safe)std::vector<server_model_meta> get_all_meta();+ struct load_options {+ server_child_mode mode = SERVER_CHILD_MODE_NORMAL;+ // used for spawning a downloading child process+ std::optional<server_model_meta> custom_meta = std::nullopt;+ };+// load and unload model instances// these functions are thread-safevoid load(const std::string & name);+ void load(const std::string & name, const load_options & opts);void unload(const std::string & name);void unload_all();- // download a new model, progress is reported via SSE- // to stop the download, call unload()- void download(common_params_model && model, common_download_opts && opts);-struct update_status_args {server_model_status status;int exit_code = 0; // only valid if status == UNLOADED@@ -213,9 +220,12 @@ public:struct server_child {// serializes the notify_to_router writesstd::mutex mtx_stdout;+ std::atomic<bool> is_finished_downloading = false; // set by run_download// return true if the current process is a child server instancebool is_child();+ server_child_mode get_mode();+ int run_download(common_params & params);// register the shutdown_handler to be called by the router// return the monitoring thread (to be joined by the caller)diff --git a/tools/server/server.cpp b/tools/server/server.cppindex bf3680b9f0..dd4b1c507c 100644--- a/tools/server/server.cpp+++ b/tools/server/server.cpp@@ -134,6 +134,7 @@ int llama_server(int argc, char ** argv) {//// register API routes+ server_child child; // only used in non-router modeserver_routes routes(params, ctx_server);server_tools tools;@@ -254,11 +255,21 @@ int llama_server(int argc, char ** argv) {ctx_http.post("/tools", ex_wrapper(tools.handle_post));}+ //+ // Handle downloading model+ //++ if (child.is_child() && child.get_mode() == SERVER_CHILD_MODE_DOWNLOAD) {+ return child.run_download(params);+ } else if (!is_router_server) {+ // single-model mode (NOT spawned by router)+ common_params_handle_models(params, LLAMA_EXAMPLE_SERVER);+ }+//// Start the server//- server_child child; // only used in non-router modestd::function<void()> clean_up;if (is_router_server) {diff --git a/tools/server/tests/unit/test_router.py b/tools/server/tests/unit/test_router.pyindex 11c77ca7aa..41e95f4c5f 100644--- a/tools/server/tests/unit/test_router.py+++ b/tools/server/tests/unit/test_router.py@@ -257,14 +257,25 @@ def test_router_reload_models():MODEL_DOWNLOAD_ID = "ggml-org/test-model-router-download:F16"-MODEL_DOWNLOAD_TIMEOUT = 300+MODEL_DOWNLOAD_TIMEOUT = 30-def _listen_sse(server: ServerProcess, collected: list, stop: threading.Event):- """Collect /models/sse events into `collected` until `stop` is set."""+def _listen_sse(+ server: ServerProcess, collected: list, stop: threading.Event, ready: threading.Event | None = None+):+ """Collect /models/sse events into `collected` until `stop` is set.++ When `ready` is provided, it is set once the streaming response is open,+ i.e. the server has accepted the connection and registered us as a+ subscriber. Callers that trigger one-shot events (e.g. download_finished)+ must wait on `ready` before acting, otherwise the event can be broadcast+ before this client is subscribed and be lost.+ """url = f"http://{server.server_host}:{server.server_port}/models/sse"try:with requests.get(url, stream=True, timeout=MODEL_DOWNLOAD_TIMEOUT) as resp:+ if ready is not None:+ ready.set()for line_bytes in resp.iter_lines():if stop.is_set():break@@ -294,11 +305,17 @@ def test_router_download_model():sse_events: list = []stop = threading.Event()+ sse_ready = threading.Event()sse_thread = threading.Thread(- target=_listen_sse, args=(server, sse_events, stop), daemon=True+ target=_listen_sse, args=(server, sse_events, stop, sse_ready), daemon=True)sse_thread.start()+ # wait for the SSE client to be subscribed before triggering the download,+ # otherwise the one-shot download_finished event can be broadcast before+ # this client is registered and be lost+ assert sse_ready.wait(10), "SSE client failed to connect"+# Trigger the downloadres = server.make_request("POST", "/models", data={"model": MODEL_DOWNLOAD_ID})assert res.status_code == 200@@ -328,13 +345,17 @@ def test_router_delete_model():# Ensure the model exists (download it if needed)if MODEL_DOWNLOAD_ID not in _get_model_ids(is_reload=False):- res = server.make_request("POST", "/models", data={"model": MODEL_DOWNLOAD_ID})- assert res.status_code == 200sse_events: list = []stop = threading.Event()+ sse_ready = threading.Event()threading.Thread(- target=_listen_sse, args=(server, sse_events, stop), daemon=True+ target=_listen_sse, args=(server, sse_events, stop, sse_ready), daemon=True).start()+ # subscribe before triggering the download so the one-shot+ # download_finished event is not lost (see test_router_download_model)+ assert sse_ready.wait(10), "SSE client failed to connect"+ res = server.make_request("POST", "/models", data={"model": MODEL_DOWNLOAD_ID})+ assert res.status_code == 200finished = _wait_for_sse_event(sse_events, "download_finished", MODEL_DOWNLOAD_ID, MODEL_DOWNLOAD_TIMEOUT)