LLM inference in C/C++
C++ ggml
server: (router) add model management API (#23976)
Xuan-Son Nguyen4b4d13ae721e
diff --git a/common/download.cpp b/common/download.cppindex 40f6eb780f..c3c8ff49bb 100644--- a/common/download.cpp+++ b/common/download.cpp@@ -997,3 +997,87 @@ std::vector<common_cached_model_info> common_list_cached_models() {return result;}++bool common_download_remove(const std::string & hf_repo_with_tag) {+ namespace fs = std::filesystem;++ auto [repo_id, tag] = common_download_split_repo_tag(hf_repo_with_tag);++ if (tag.empty()) {+ return hf_cache::remove_cached_repo(repo_id);+ }++ std::string tag_upper = tag;+ for (char & c : tag_upper) {+ c = (char) std::toupper((unsigned char) c);+ }++ auto files = hf_cache::get_cached_files(repo_id);+ if (files.empty()) {+ return false;+ }++ // collect snapshot entries whose tag matches+ std::vector<fs::path> to_remove;+ for (const auto & f : files) {+ auto split = get_gguf_split_info(f.path);+ if (split.tag == tag_upper) {+ to_remove.emplace_back(f.local_path);+ }+ }++ if (to_remove.empty()) {+ return false;+ }++ // resolve blob paths from symlinks before deleting snapshot entries+ std::vector<fs::path> blobs_to_check;+ for (const auto & p : to_remove) {+ std::error_code ec;+ if (fs::is_symlink(p, ec)) {+ auto target = fs::read_symlink(p, ec);+ if (!ec) {+ blobs_to_check.push_back((p.parent_path() / target).lexically_normal());+ }+ }+ }++ // remove snapshot entries+ for (const auto & p : to_remove) {+ std::error_code ec;+ fs::remove(p, ec);+ if (ec) {+ LOG_WRN("%s: failed to remove %s: %s\n", __func__, p.string().c_str(), ec.message().c_str());+ }+ }++ if (blobs_to_check.empty()) {+ return true;+ }++ // collect blobs still referenced by remaining snapshot entries+ std::unordered_set<std::string> still_referenced;+ for (const auto & f : hf_cache::get_cached_files(repo_id)) {+ fs::path p(f.local_path);+ std::error_code ec;+ if (fs::is_symlink(p, ec)) {+ auto target = fs::read_symlink(p, ec);+ if (!ec) {+ still_referenced.insert((p.parent_path() / target).lexically_normal().string());+ }+ }+ }++ // remove orphaned blobs+ for (const auto & blob : blobs_to_check) {+ if (still_referenced.find(blob.string()) == still_referenced.end()) {+ std::error_code ec;+ fs::remove(blob, ec);+ if (ec) {+ LOG_WRN("%s: failed to remove blob %s: %s\n", __func__, blob.string().c_str(), ec.message().c_str());+ }+ }+ }++ return true;+}diff --git a/common/download.h b/common/download.hindex ebeedd6058..2371797644 100644--- a/common/download.h+++ b/common/download.h@@ -115,3 +115,10 @@ int common_download_file_single(const std::string & url,// resolve and download model from Docker registry// return local path to downloaded model filestd::string common_docker_resolve_model(const std::string & docker);++// Remove a cached model from disk+// input format: "user/model" or "user/model:tag"+// - if tag is omitted, removes the entire repo cache directory+// - if tag is present, removes only files matching that tag (and orphaned blobs)+// returns true if anything was removed+bool common_download_remove(const std::string & hf_repo_with_tag);diff --git a/common/hf-cache.cpp b/common/hf-cache.cppindex ba7417a12b..f1dacaa477 100644--- a/common/hf-cache.cpp+++ b/common/hf-cache.cpp@@ -495,4 +495,19 @@ std::string finalize_file(const hf_file & file) {return file.final_path;}+bool remove_cached_repo(const std::string & repo_id) {+ if (!is_valid_repo_id(repo_id)) {+ LOG_WRN("%s: invalid repository: %s\n", __func__, repo_id.c_str());+ return false;+ }+ fs::path repo_path = get_repo_path(repo_id);+ std::error_code ec;+ auto removed = fs::remove_all(repo_path, ec);+ if (ec) {+ LOG_ERR("%s: failed to remove repo cache %s: %s\n", __func__, repo_path.string().c_str(), ec.message().c_str());+ return false;+ }+ return removed > 0;+}+} // namespace hf_cachediff --git a/common/hf-cache.h b/common/hf-cache.hindex 23fa0adb72..42c9c6ce34 100644--- a/common/hf-cache.h+++ b/common/hf-cache.h@@ -29,4 +29,7 @@ hf_files get_cached_files(const std::string & repo_id = {});// Create snapshot path (link or move/copy) and return itstd::string finalize_file(const hf_file & file);+// Remove the entire cached directory for a repo, returns true if removed+bool remove_cached_repo(const std::string & repo_id);+} // namespace hf_cachediff --git a/tools/server/README-dev.md b/tools/server/README-dev.mdindex 0ff334724a..4c41031239 100644--- a/tools/server/README-dev.md+++ b/tools/server/README-dev.md@@ -180,6 +180,24 @@ That requires `JSON.stringify` when formatted to message content:}```+### Model management API (router mode)++Model management API was added via PR [#23976](https://github.com/ggml-org/llama.cpp/pull/23976)++The main goal of this API is to allow downloading models and/or removing models from the web UI. It relies on the model cache infrastructure under the hood to manage the list of models dynamically.++Instead of building everything from the ground up (like what most AI agents will do when you ask them to implement a similar feature), we built on top of existing, already well-engineered components inside the codebase:+- Model cache infrastructure as mentioned above (`common/download.h`)+- Server response queue (`server-queue.h`). We use this feature to broadcast events to SSE clients.+- Server router thread management (`server-models.h`). We re-use the same thread model that is used for managing subprocess life cycle, except that we don't create a new subprocess, but launch the download right inside the thread.++The flow for downloading a new model:+- POST request comes in --> `post_router_models` --> validation+- `server_models::download()` is called+ - Sets up a new thread `inst.th` and runs the download inside+- If a stop request comes in, set `stop_download` to `true`+- Otherwise, upon completion, we call `load_models()` to refresh the list of models+### Notable Related PRs- Initial server implementation: https://github.com/ggml-org/llama.cpp/pull/1443diff --git a/tools/server/README.md b/tools/server/README.mdindex b414910590..88a507e2c5 100644--- a/tools/server/README.md+++ b/tools/server/README.md@@ -1778,6 +1778,20 @@ The `status` object can be:}```+Note: for "downloading" state, there can be multiple files be downloading in parallel++```json+"status": {+ "value": "downloading",+ "progress": {+ "https://...model.gguf": {+ "done": 195963406,+ "total": 219307424+ }+ }+}+```+### POST `/models/load`: Load a modelLoad a model@@ -1820,6 +1834,107 @@ Response:}```+### GET `/models/sse`: Real-time events++Example events:++```js+{+ "model": "...",+ "event": "model_status",+ "data": {+ "status": "loading"+ }+}++{+ "model": "...",+ "event": "download_progress",+ "data": {+ // note: there can be multiple files being downloaded in parallel+ "https://...model.gguf": {+ "done": 195963406,+ "total": 219307424+ }+ }+}++{+ "model": "...",+ "event": "download_finished",+ "data": {+ "status": "loading"+ }+}++{+ "model": "...",+ "event": "model_remove"+}++// special event: reload of the list of all models+{+ "model": "*",+ "event": "models_reload"+}+```++### POST `/models`: Download new model++Trigger a new download (non-blocking), the progress can be tracked via SSE endpoint `/models/sse`++To cancel model downloading, send an event to `/models/unload`++Download procedure:+- Send POST request to `/models`+- Subscribe to `/models/sse` for updates+- On downloading completed, you will receive either `download_finished` or `download_failed` event+- Call GET `/models` to trigger model list update. If the download success, you should see the new model in the list++Payload:++```json+{+ "model": "ggml-org/gemma-3-4b-it-GGUF:Q4_K_M",+}+```++Response (download is started in the background):++```json+{+ "success": true+}+```++Response (error, cannot start the download):++```json+{+ "error": {+ "code": 400,+ "message": "model validation failed, unable to download",+ "type": "invalid_request_error"+ }+}+```++### DELETE `/models`: Delete a model from cache++IMPORTANT: only model stored in cache can be deleted. You cannot delete models in a preset.++Model name must be passed via query param: `?model={name}`++If delete success, it will send an SSE event of type `model_remove`++Response:++```json+{+ "success": true+}+```+## API errors`llama-server` returns errors in the same format as OAI: https://github.com/openai/openai-openapidiff --git a/tools/server/server-http.cpp b/tools/server/server-http.cppindex 3c58fcfece..5defee1f5e 100644--- a/tools/server/server-http.cpp+++ b/tools/server/server-http.cpp@@ -588,6 +588,23 @@ void server_http_context::post(const std::string & path, const server_http_conte});}+void server_http_context::del(const std::string & path, const server_http_context::handler_t & handler) const {+ handlers.emplace(path, handler);+ pimpl->srv->Delete(path_prefix + path, [handler](const httplib::Request & req, httplib::Response & res) {+ server_http_req_ptr request = std::make_unique<server_http_req>(server_http_req{+ get_params(req),+ get_headers(req),+ req.path,+ build_query_string(req),+ req.body,+ {},+ req.is_connection_closed+ });+ server_http_res_ptr response = handler(*request);+ process_handler_response(std::move(request), response, res);+ });+}+//// Vertex AI Prediction protocol (AIP_PREDICT_ROUTE)// https://cloud.google.com/vertex-ai/docs/predictions/custom-container-requirementsdiff --git a/tools/server/server-http.h b/tools/server/server-http.hindex 25c7f10629..6b4a4b87a6 100644--- a/tools/server/server-http.h+++ b/tools/server/server-http.h@@ -86,6 +86,7 @@ struct server_http_context {void get(const std::string & path, const handler_t & handler) const;void post(const std::string & path, const handler_t & handler) const;+ void del(const std::string & path, const handler_t & handler) const;// Register the Google Cloud Platform (Vertex AI) compat (AIP_PREDICT_ROUTE env var, or /predict)// Must be called AFTER all other API routes are registereddiff --git a/tools/server/server-models.cpp b/tools/server/server-models.cppindex 49b0e423f4..ff9a0df12f 100644--- a/tools/server/server-models.cpp+++ b/tools/server/server-models.cpp@@ -9,6 +9,7 @@#include <sheredom/subprocess.h>#include <functional>+#include <optional>#include <algorithm>#include <thread>#include <mutex>@@ -51,6 +52,21 @@ extern char **environ;// ref: https://github.com/ggml-org/llama.cpp/issues/17862#define CHILD_ADDR "127.0.0.1"+struct server_subproc {+ std::optional<subprocess_s> sproc; // empty while in DOWNLOADING state+ std::atomic<bool> stop_download{false}; // flag to signal download cancellation++ subprocess_s & get() {+ GGML_ASSERT(sproc.has_value() && "subprocess not initialized");+ return sproc.value();+ }++ bool is_alive() {+ return sproc.has_value() && subprocess_alive(&sproc.value());+ }+};++static std::filesystem::path get_server_exec_path() {#if defined(_WIN32)wchar_t buf[32768] = { 0 }; // Large buffer to handle long paths@@ -272,12 +288,25 @@ void server_models::add_model(server_model_meta && meta) {meta.update_caps();std::string name = meta.name;mapping[name] = instance_t{- /* subproc */ std::make_shared<subprocess_s>(),+ /* subproc */ std::make_shared<server_subproc>(),/* th */ std::thread(),/* meta */ std::move(meta)};}+void server_models::notify_sse(const std::string & event, const std::string & model_id, const json & data) {+ std::unique_ptr<server_task_result_router> result = std::make_unique<server_task_result_router>();+ result->data = {+ {"model", model_id},+ {"event", event},+ };+ if (!data.is_null()) {+ result->data["data"] = data;+ }+ SRV_DBG("notifying SSE clients about event '%s' for model '%s': %s\n", event.c_str(), model_id.c_str(), safe_json_to_str(result->data).c_str());+ sse.broadcast(std::move(result));+}+void server_models::load_models() {// Phase 1: load presets from all sources — pure I/O, no lock needed// 1. cached models@@ -304,19 +333,27 @@ void server_models::load_models() {// note: if a model exists in both cached and local, local takes precedencecommon_presets final_presets;- for (const auto & [name, preset] : cached_models) final_presets[name] = preset;- for (const auto & [name, preset] : local_models) final_presets[name] = preset;+ std::unordered_map<std::string, server_model_source> source_map;+ for (const auto & [name, preset] : cached_models) {+ final_presets[name] = preset;+ source_map[name] = SERVER_MODEL_SOURCE_CACHE;+ }+ for (const auto & [name, preset] : local_models) {+ final_presets[name] = preset;+ source_map[name] = SERVER_MODEL_SOURCE_MODELS_DIR;+ }for (const auto & [name, custom] : custom_presets) {if (final_presets.find(name) != final_presets.end()) {final_presets[name].merge(custom);} else {final_presets[name] = custom;}+ source_map[name] = SERVER_MODEL_SOURCE_PRESET;}- // server base preset from CLI args takes highest precedence- for (auto & [name, preset] : final_presets) {- preset.merge(base_preset);- }++ auto get_source = [&](const std::string & name) {+ return source_map.count(name) ? source_map.at(name) : SERVER_MODEL_SOURCE_PRESET;+ };// Helpers that read `mapping` — must be called while holding the lock.std::unordered_set<std::string> custom_names;@@ -366,12 +403,15 @@ void server_models::load_models() {// (unload, load) or when joining threads (the monitoring thread calls update_status// which locks the mutex, so joining while holding it would deadlock).std::unique_lock<std::mutex> lk(mutex);++ need_reload = false;bool is_first_load = mapping.empty();if (is_first_load) {// FIRST LOAD: add all models, then unlock for autoloadingfor (const auto & [name, preset] : final_presets) {server_model_meta meta{+ /* source */ get_source(name),/* preset */ preset,/* name */ name,/* aliases */ {},@@ -384,7 +424,7 @@ void server_models::load_models() {/* exit_code */ 0,/* stop_timeout */ DEFAULT_STOP_TIMEOUT,/* multimodal */ mtmd_caps{false, false},- /* need_download */ false,+ // /* need_download */ false,};add_model(std::move(meta));}@@ -453,6 +493,9 @@ void server_models::load_models() {}}for (auto & [name, inst] : mapping) {+ if (inst.meta.status == SERVER_MODEL_STATUS_DOWNLOADING) {+ continue; // downloading models are not from config sources, leave them alone+ }if (final_presets.find(name) == final_presets.end() && !inst.meta.is_running() && inst.th.joinable()) {threads_to_join.push_back(std::move(inst.th));}@@ -465,7 +508,15 @@ void server_models::load_models() {// erase models no longer in any sourcefor (auto it = mapping.begin(); it != mapping.end(); ) {- if (final_presets.find(it->first) == final_presets.end()) {+ if (it->second.meta.status == SERVER_MODEL_STATUS_DOWNLOADING) {+ ++it; // download thread is still busy, skip+ } else if (it->second.meta.status == SERVER_MODEL_STATUS_DOWNLOADED) {+ // download finished, safe to erase+ if (it->second.th.joinable()) {+ it->second.th.join();+ }+ it = mapping.erase(it);+ } else if (final_presets.find(it->first) == final_presets.end()) {SRV_INF("(reload) removing model name=%s (no longer in source)\n", it->first.c_str());GGML_ASSERT(!it->second.th.joinable()); // must have been joined aboveit = mapping.erase(it);@@ -526,6 +577,7 @@ void server_models::load_models() {for (const auto & [name, preset] : final_presets) {if (mapping.find(name) == mapping.end()) {server_model_meta meta{+ /* source */ get_source(name),/* preset */ preset,/* name */ name,/* aliases */ {},@@ -538,7 +590,7 @@ void server_models::load_models() {/* exit_code */ 0,/* stop_timeout */ DEFAULT_STOP_TIMEOUT,/* multimodal */ mtmd_caps{false, false},- /* need_download */ false,+ // /* need_download */ false,};add_model(std::move(meta));newly_added.push_back(name);@@ -571,6 +623,8 @@ void server_models::load_models() {SRV_INF("(reload) loading new model %s\n", name.c_str());load(name);}++ notify_sse("models_reload", "*");}}@@ -597,7 +651,13 @@ bool server_models::has_model(const std::string & name) {}std::optional<server_model_meta> server_models::get_meta(const std::string & name) {- std::lock_guard<std::mutex> lk(mutex);+ std::unique_lock<std::mutex> lk(mutex);+ if (need_reload) {+ lk.unlock();+ load_models();+ lk.lock();+ }+auto it = mapping.find(name);if (it != mapping.end()) {return it->second.meta;@@ -683,7 +743,13 @@ static std::vector<char *> to_char_ptr_array(const std::vector<std::string> & ve}std::vector<server_model_meta> server_models::get_all_meta() {- std::lock_guard<std::mutex> lk(mutex);+ std::unique_lock<std::mutex> lk(mutex);+ if (need_reload) {+ lk.unlock();+ load_models();+ lk.lock();+ }+std::vector<server_model_meta> result;result.reserve(mapping.size());for (const auto & [name, inst] : mapping) {@@ -770,7 +836,7 @@ void server_models::load(const std::string & name) {throw std::runtime_error("failed to get a port number");}- inst.subproc = std::make_shared<subprocess_s>();+ inst.subproc = std::make_shared<server_subproc>();{SRV_INF("spawning server instance with name=%s on port %d\n", inst.meta.name.c_str(), inst.meta.port);@@ -792,19 +858,20 @@ void server_models::load(const std::string & name) {// TODO @ngxson : maybe separate stdout and stderr in the future// so that we can use stdout for commands and stderr for loggingint options = subprocess_option_no_window | subprocess_option_combined_stdout_stderr;- int result = subprocess_create_ex(argv.data(), options, envp.data(), inst.subproc.get());+ inst.subproc->sproc.emplace();+ int result = subprocess_create_ex(argv.data(), options, envp.data(), &inst.subproc->get());if (result != 0) {throw std::runtime_error("failed to spawn server instance");}- inst.stdin_file = subprocess_stdin(inst.subproc.get());+ inst.stdin_file = subprocess_stdin(&inst.subproc->get());}// start a thread to manage the child process// captured variables are guaranteed to be destroyed only after the thread is joinedinst.th = std::thread([this, name, child_proc = inst.subproc, port = inst.meta.port, stop_timeout = inst.meta.stop_timeout]() {- FILE * stdin_file = subprocess_stdin(child_proc.get());- FILE * stdout_file = subprocess_stdout(child_proc.get()); // combined stdout/stderr+ FILE * stdin_file = subprocess_stdin(&child_proc->get());+ FILE * stdout_file = subprocess_stdout(&child_proc->get()); // combined stdout/stderrstd::thread log_thread([&]() {// read stdout/stderr and forward to main server log@@ -834,14 +901,14 @@ void server_models::load(const std::string & name) {return this->stopping_models.find(name) != this->stopping_models.end();};auto should_wake = [&]() {- return is_stopping() || !subprocess_alive(child_proc.get());+ return is_stopping() || !child_proc->is_alive();};{std::unique_lock<std::mutex> lk(this->mutex);this->cv_stop.wait(lk, should_wake);}// child may have already exited (e.g. crashed) — skip shutdown sequence- if (!subprocess_alive(child_proc.get())) {+ if (!child_proc->is_alive()) {return;}SRV_INF("stopping model instance name=%s\n", name.c_str());@@ -859,7 +926,7 @@ void server_models::load(const std::string & name) {if (elapsed >= stop_timeout * 1000) {// timeout, force killSRV_WRN("force-killing model instance name=%s after %d seconds timeout\n", name.c_str(), stop_timeout);- subprocess_terminate(child_proc.get());+ subprocess_terminate(&child_proc->get());return;}this->cv_stop.wait_for(lk, std::chrono::seconds(1));@@ -884,8 +951,8 @@ void server_models::load(const std::string & name) {// get the exit codeint exit_code = 0;- subprocess_join(child_proc.get(), &exit_code);- subprocess_destroy(child_proc.get());+ subprocess_join(&child_proc->get(), &exit_code);+ subprocess_destroy(&child_proc->get());// update status and exit codethis->update_status(name, SERVER_MODEL_STATUS_UNLOADED, exit_code);@@ -896,30 +963,118 @@ void server_models::load(const std::string & name) {{auto & old_instance = mapping[name];// old process should have exited already, but just in case, we clean it up here- if (subprocess_alive(old_instance.subproc.get())) {+ if (old_instance.subproc->is_alive()) {SRV_WRN("old process for model name=%s is still alive, this is unexpected\n", name.c_str());- subprocess_terminate(old_instance.subproc.get()); // force kill+ subprocess_terminate(&old_instance.subproc->get()); // force kill}if (old_instance.th.joinable()) {old_instance.th.join();}}+ notify_sse("model_status", name, {+ {"status", server_model_status_to_string(inst.meta.status)},+ });+mapping[name] = std::move(inst);cv.notify_all();}+// callback for model downloading functionality+struct server_models_download_res : public common_download_callback {+ common_params_model model;+ common_download_opts opts;++ std::function<bool()> should_stop;+ std::function<void(const common_download_progress & p)> on_progress;++ bool is_ok = false;++ bool run() {+ try {+ common_download_model(model, opts);+ is_ok = true;+ } catch (const std::exception & e) {+ SRV_ERR("download failed for model name=%s: %s\n", model.name.c_str(), e.what());+ is_ok = false;+ }+ return is_ok;+ }+ void on_start(const common_download_progress & p) override {+ on_progress(p);+ }+ void on_update(const common_download_progress & p) override {+ on_progress(p);+ }+ void on_done(const common_download_progress &, bool ok) override {+ is_ok = ok;+ }+ bool is_cancelled() const override {+ return should_stop();+ }+};++void server_models::download(common_params_model && model, common_download_opts && opts) {+ std::string name = model.name;+ GGML_ASSERT(name == model.hf_repo);++ std::unique_lock<std::mutex> lk(mutex);+ if (mapping.find(name) != mapping.end()) {+ throw std::runtime_error("model name=" + name + " already exists");+ }++ instance_t inst;+ inst.meta.name = name;+ inst.meta.status = SERVER_MODEL_STATUS_DOWNLOADING;+ inst.subproc = std::make_shared<server_subproc>();++ auto dl = std::make_unique<server_models_download_res>();+ dl->model = model; // copy+ dl->opts = opts; // copy++ dl->should_stop = [sp = inst.subproc]() {+ return sp->stop_download.load(std::memory_order_relaxed);+ };++ dl->on_progress = [this, name](const common_download_progress & p) {+ update_download_progress(name, p, false);+ };++ inst.th = std::thread([this, dl = std::move(dl)]() {+ dl->opts.callback = dl.get();+ bool ok = dl->run();+ SRV_INF("download finished for model name=%s with status=%s\n",+ dl->model.name.c_str(), ok ? "success" : "failure");+ update_download_progress(dl->model.name, {}, true, ok);+ // need_reload is set inside update_download_progress under the mutex;+ // the next load_models() call will clean up this instance+ });++ mapping[name] = std::move(inst);+ notify_sse("status_update", name, {+ {"status", server_model_status_to_string(SERVER_MODEL_STATUS_DOWNLOADING)},+ });+ cv.notify_all();+}+void server_models::unload(const std::string & name) {- std::lock_guard<std::mutex> lk(mutex);+ std::unique_lock<std::mutex> lk(mutex);auto it = mapping.find(name);if (it != mapping.end()) {- if (it->second.meta.is_running()) {+ if (it->second.meta.status == SERVER_MODEL_STATUS_DOWNLOADING) {+ SRV_INF("cancelling download for model name=%s\n", name.c_str());+ it->second.subproc->stop_download.store(true, std::memory_order_relaxed);+ // for convenience, we wait the status change here+ wait(lk, name, [](const server_model_meta & new_meta) {+ return new_meta.status != SERVER_MODEL_STATUS_DOWNLOADING;+ });+ } else if (it->second.meta.is_running()) {SRV_INF("stopping model instance name=%s\n", name.c_str());stopping_models.insert(name);if (it->second.meta.status == SERVER_MODEL_STATUS_LOADING) {// special case: if model is in loading state, unloading means force-killing itSRV_WRN("model name=%s is still loading, force-killing\n", name.c_str());- subprocess_terminate(it->second.subproc.get());+ subprocess_terminate(&it->second.subproc->get());}cv_stop.notify_all();// status change will be handled by the managing thread@@ -934,7 +1089,10 @@ void server_models::unload_all() {{std::lock_guard<std::mutex> lk(mutex);for (auto & [name, inst] : mapping) {- if (inst.meta.is_running()) {+ if (inst.meta.status == SERVER_MODEL_STATUS_DOWNLOADING) {+ SRV_INF("cancelling download for model name=%s\n", name.c_str());+ inst.subproc->stop_download.store(true, std::memory_order_relaxed);+ } else if (inst.meta.is_running()) {SRV_INF("stopping model instance name=%s\n", name.c_str());stopping_models.insert(name);cv_stop.notify_all();@@ -959,6 +1117,17 @@ void server_models::update_status(const std::string & name, server_model_statusmeta.status = status;meta.exit_code = exit_code;}+ // broadcast status change to SSE+ {+ json data = {+ {"status", server_model_status_to_string(status)},+ };+ if (status == SERVER_MODEL_STATUS_UNLOADED) {+ data["exit_code"] = exit_code;+ }+ // note: notify_sse doesn't acquire the lock, so no deadlock here+ notify_sse("status_change", name, data);+ }cv.notify_all();}@@ -985,12 +1154,82 @@ void server_models::update_loaded_info(const std::string & name, std::string & rcv.notify_all();}-void server_models::wait_until_loading_finished(const std::string & name) {+void server_models::update_download_progress(const std::string & name, const common_download_progress & progress, bool done, bool ok) {+ json curr;+ {+ std::lock_guard<std::mutex> lk(mutex);+ auto it = mapping.find(name);+ if (it != mapping.end()) {+ if (done) {+ // mark the instance to be erased on next load_models() call+ it->second.meta.status = SERVER_MODEL_STATUS_DOWNLOADED;+ need_reload = true;+ } else {+ json & info = it->second.meta.loaded_info;+ if (!info.contains("progress")) {+ info["progress"] = json{};+ }+ info["progress"][progress.url] = {+ {"done", progress.downloaded},+ {"total", progress.total},+ };+ curr = it->second.meta.loaded_info; // copy+ }+ }+ }+ if (done) {+ cv.notify_all(); // notify in case unload() is waiting for download to be cancelled+ notify_sse(ok ? "download_finished" : "download_failed", name, {});+ } else {+ notify_sse("download_progress", name, curr);+ }+}++bool server_models::remove(const std::string & name) {+ auto meta = get_meta(name);++ if (!meta.has_value()) {+ throw std::runtime_error("model name=" + name + " is not found");+ }+ if (meta->source != SERVER_MODEL_SOURCE_CACHE) {+ throw std::runtime_error("model name=" + name + " is not removable (not from cache)");+ }++ unload(name); // cancel download or stop running instance+ {+ std::unique_lock<std::mutex> lk(mutex);+ // a cancelled download lands on DOWNLOADED; a stopped instance lands on UNLOADED+ wait(lk, name, [](const server_model_meta & new_meta) {+ return new_meta.status == SERVER_MODEL_STATUS_UNLOADED+ || new_meta.status == SERVER_MODEL_STATUS_DOWNLOADED;+ });+ // join before erasing - after status reaches UNLOADED/DOWNLOADED the thread no+ // longer acquires this mutex, so joining while holding it is safe+ if (mapping[name].th.joinable()) {+ mapping[name].th.join();+ }+ // remove the model from disk (hold lock to prevent concurrent load)+ bool ok = common_download_remove(name);+ if (ok) {+ mapping.erase(name);+ }+ SRV_INF("removing model name=%s from cache (%s)\n", name.c_str(), ok ? "succeeded" : "failed");+ notify_sse("model_remove", name, {});+ return ok;+ }+}++void server_models::wait(const std::string & name, std::function<bool(const server_model_meta &)> predicate) {std::unique_lock<std::mutex> lk(mutex);- cv.wait(lk, [this, &name]() {+ wait(lk, name, predicate);+}++void server_models::wait(std::unique_lock<std::mutex> & lk, const std::string & name, std::function<bool(const server_model_meta &)> predicate) {+ cv.wait(lk, [this, &name, &predicate]() {auto it = mapping.find(name);if (it != mapping.end()) {- return it->second.meta.status != SERVER_MODEL_STATUS_LOADING;+ return predicate(it->second.meta);+}return false;});@@ -1014,10 +1253,15 @@ bool server_models::ensure_model_ready(const std::string & name) {// wait for loading to completeSRV_INF("waiting until model name=%s is fully loaded...\n", name.c_str());- wait_until_loading_finished(name);+ wait(name, [&meta](const server_model_meta & new_meta) {+ if (new_meta.status != SERVER_MODEL_STATUS_LOADING) {+ meta = new_meta; // update meta for final check after wait+ return true;+ }+ return false;+ });// check final status- meta = get_meta(name);if (!meta.has_value() || meta->is_failed()) {throw std::runtime_error("model name=" + name + " failed to load");}@@ -1111,6 +1355,42 @@ void server_models::notify_router_sleeping_state(bool is_sleeping) {// server_models_routes//+// RAII wrapper similar to server_response_reader, but doesn't use server_queue+static std::atomic<int> sse_client_id_counter = 0;+struct server_models_sse_client {+ server_response & queue_results;+ int client_id;+ server_models_sse_client(server_response & q)+ : queue_results(q), client_id(sse_client_id_counter.fetch_add(1, std::memory_order_relaxed)) {+ SRV_DBG("new SSE client connected, assigned client_id=%d\n", client_id);+ queue_results.add_waiting_task_id(client_id);+ }+ ~server_models_sse_client() {+ SRV_DBG("SSE client disconnected, removing client_id=%d\n", client_id);+ queue_results.remove_waiting_task_id(client_id);+ }++ // return nullptr if should_stop() is true before receiving a result+ // note: if one error is received, it will stop further processing and return error result+ server_task_result_ptr next(const std::function<bool()> & should_stop) {+ while (true) {+ static const int http_polling_seconds = 1; // check should_stop every 1 second+ server_task_result_ptr result = queue_results.recv_with_timeout({client_id}, http_polling_seconds);+ if (result == nullptr) {+ // timeout, check stop condition+ if (should_stop()) {+ return nullptr;+ }+ // continue waiting otherwise+ } else {+ SRV_DBG("recv result for client_id=%d: %s\n", client_id, safe_json_to_str(result->to_json()).c_str());+ return result;+ }+ }+ // should not reach here+ }+};+static void res_ok(std::unique_ptr<server_http_res> & res, const json & response_data) {res->status = 200;res->data = safe_json_to_str(response_data);@@ -1274,7 +1554,9 @@ void server_models_routes::init_routes() {{"created", t}, // for OAI-compat{"status", status},{"architecture", architecture},- {"need_download", meta.need_download},+ {"source", server_model_source_to_string(meta.source)},+ {"can_remove", meta.source == SERVER_MODEL_SOURCE_CACHE},+ // {"need_download", meta.need_download},// TODO: add other fields, may require reading GGUF metadata};@@ -1312,6 +1594,87 @@ void server_models_routes::init_routes() {res_ok(res, {{"success", true}});return res;};++ this->get_router_models_sse = [this](const server_http_req & req) {+ auto res = std::make_unique<server_http_res>();+ res->status = 200;+ res->content_type = "text/event-stream";+ auto sse_client = std::make_shared<server_models_sse_client>(models.sse);+ res->next = [this, sse_client, &req](std::string & output) -> bool {+ auto result = sse_client->next([&]() {+ return stopping.load(std::memory_order_relaxed) || req.should_stop();+ });+ if (result == nullptr) {+ return false; // client disconnected or should_stop+ }+ output = "data: " + safe_json_to_str(result->to_json()) + "\n\n";+ return true; // listen for the next event+ };+ return res;+ };++ this->post_router_models = [this](const server_http_req & req) {+ auto res = std::make_unique<server_http_res>();++ json body = json::parse(req.body);+ std::string name = json_value(body, "model", std::string());+ if (name.empty()) {+ throw std::invalid_argument("model must be a non-empty string");+ }++ common_params_model model;+ common_download_opts opts;++ model.name = name;+ model.hf_repo = name;+ opts.bearer_token = params.hf_token;+ opts.download_mmproj = true;+ opts.download_mtp = true;++ // first, only check if the model is valid and can be downloaded+ opts.skip_download = true;+ bool ok = false;+ try {+ auto validation = common_download_model(model, opts);+ ok = !validation.model_path.empty();+ } catch (const common_skip_download_exception &) {+ // model is valid and will be downloaded+ ok = true;+ } catch (...) {+ SRV_ERR("unknown error while validating model '%s'\n", name.c_str());+ // other exceptions will be handled by the outer ex_wrapper()+ throw;+ }++ if (!ok) {+ throw std::invalid_argument("model validation failed, unable to download");+ }++ // then, proceed with the actual download+ opts.skip_download = false;+ SRV_INF("starting download for model '%s'\n", name.c_str());+ models.download(std::move(model), std::move(opts));++ res_ok(res, {{"success", true}});+ return res;+ };++ this->del_router_models = [this](const server_http_req & req) {+ auto res = std::make_unique<server_http_res>();++ std::string name = req.get_param("model");+ if (name.empty()) {+ throw std::invalid_argument("model must be a non-empty string");+ }++ bool ok = models.remove(name);+ if (!ok) {+ throw std::runtime_error("failed to remove model '" + name + "'");+ }++ res_ok(res, {{"success", true}});+ return res;+ };}diff --git a/tools/server/server-models.h b/tools/server/server-models.hindex 2198589a7a..319c4352e2 100644--- a/tools/server/server-models.h+++ b/tools/server/server-models.h@@ -1,9 +1,11 @@#pragma once#include "common.h"+#include "download.h"#include "preset.h"#include "server-common.h"#include "server-http.h"+#include "server-queue.h"#include <mutex>#include <condition_variable>@@ -14,6 +16,8 @@/*** state diagram:*+ * DOWNLOADING ──► DOWNLOADED ──► (replaced by new instance)+ ** UNLOADED ──► LOADING ──► LOADED ◄──── SLEEPING* ▲ │ │ ▲* └───failed───┘ │ │@@ -22,39 +26,43 @@*/enum server_model_status {// TODO: also add downloading state when the logic is added+ SERVER_MODEL_STATUS_DOWNLOADING,+ SERVER_MODEL_STATUS_DOWNLOADED,SERVER_MODEL_STATUS_UNLOADED,SERVER_MODEL_STATUS_LOADING,SERVER_MODEL_STATUS_LOADED,SERVER_MODEL_STATUS_SLEEPING};-static server_model_status server_model_status_from_string(const std::string & status_str) {- if (status_str == "unloaded") {- return SERVER_MODEL_STATUS_UNLOADED;- }- if (status_str == "loading") {- return SERVER_MODEL_STATUS_LOADING;- }- if (status_str == "loaded") {- return SERVER_MODEL_STATUS_LOADED;- }- if (status_str == "sleeping") {- return SERVER_MODEL_STATUS_SLEEPING;- }- throw std::runtime_error("invalid server model status");-}+enum server_model_source {+ SERVER_MODEL_SOURCE_PRESET,+ SERVER_MODEL_SOURCE_MODELS_DIR,+ SERVER_MODEL_SOURCE_CACHE,+};static std::string server_model_status_to_string(server_model_status status) {switch (status) {- case SERVER_MODEL_STATUS_UNLOADED: return "unloaded";- case SERVER_MODEL_STATUS_LOADING: return "loading";- case SERVER_MODEL_STATUS_LOADED: return "loaded";- case SERVER_MODEL_STATUS_SLEEPING: return "sleeping";- default: return "unknown";+ case SERVER_MODEL_STATUS_DOWNLOADING: return "downloading";+ case SERVER_MODEL_STATUS_DOWNLOADED: return "downloaded";+ case SERVER_MODEL_STATUS_UNLOADED: return "unloaded";+ case SERVER_MODEL_STATUS_LOADING: return "loading";+ case SERVER_MODEL_STATUS_LOADED: return "loaded";+ case SERVER_MODEL_STATUS_SLEEPING: return "sleeping";+ default: return "unknown";+ }+}++static std::string server_model_source_to_string(server_model_source source) {+ switch (source) {+ case SERVER_MODEL_SOURCE_PRESET: return "preset";+ case SERVER_MODEL_SOURCE_MODELS_DIR: return "models_dir";+ case SERVER_MODEL_SOURCE_CACHE: return "cache";+ default: return "unknown";}}struct server_model_meta {+ server_model_source source = SERVER_MODEL_SOURCE_CACHE;common_preset preset;std::string name;std::set<std::string> aliases; // additional names that resolve to this model@@ -63,11 +71,11 @@ struct server_model_meta {server_model_status status = SERVER_MODEL_STATUS_UNLOADED;int64_t last_used = 0; // for LRU unloadingstd::vector<std::string> args; // args passed to the model instance, will be populated by render_args()- json loaded_info; // info to be reflected via /v1/models endpoint+ json loaded_info; // info to be reflected via /v1/models endpoint ; if in DOWNLOADING state, it should contain download progress infoint exit_code = 0; // exit code of the model instance process (only valid if status == FAILED)int stop_timeout = 0; // seconds to wait before force-killing the model instance during shutdownmtmd_caps multimodal; // multimodal capabilities- bool need_download = false; // whether the model needs to be downloaded before loading+ // bool need_download = false; // whether the model needs to be downloaded before loading // TODO @ngxson: implement thisbool is_ready() const {return status == SERVER_MODEL_STATUS_LOADED;@@ -85,12 +93,15 @@ struct server_model_meta {void update_caps();};-struct subprocess_s;+struct server_models_routes;+struct server_subproc; // defined in server-models.cppstruct server_models {+ friend struct server_models_routes;+private:struct instance_t {- std::shared_ptr<subprocess_s> subproc; // shared between main thread and monitoring thread+ std::shared_ptr<server_subproc> subproc; // shared between main thread and monitoring threadstd::thread th;server_model_meta meta;FILE * stdin_file = nullptr;@@ -107,6 +118,9 @@ private:// set to true while load_models() is executing a reload; load() will wait until clearbool is_reloading = false;+ // if true, the next get_meta() will trigger a reload of model list+ bool need_reload = false;+common_preset_context ctx_preset;common_params base_params;@@ -122,9 +136,14 @@ private:// not thread-safe, caller must hold mutexvoid add_model(server_model_meta && meta);+ // notify SSE clients+ void notify_sse(const std::string & event, const std::string & model_id, const json & data = nullptr);+public:server_models(const common_params & params, int argc, char ** argv);+ server_response sse; // for real-time updates via SSE endpoint+// (re-)load the list of models from various sources and prepare the metadata mapping// - if this is called the first time, simply populate the metadata// - if this is called subsequently (e.g. when refreshing from disk):@@ -147,13 +166,24 @@ public:void unload(const std::string & name);void unload_all();+ // download a new model, progress is reported via SSE+ // to stop the download, call unload()+ void download(common_params_model && model, common_download_opts && opts);+// update the status of a model instance (thread-safe)void update_status(const std::string & name, server_model_status status, int exit_code);void update_loaded_info(const std::string & name, std::string & raw_info);+ void update_download_progress(const std::string & name, const common_download_progress & progress, bool done, bool ok = true);++ // remove a cache model from disk and update the list (thread-safe)+ // note: only cache models can be removed; returns false if the model doesn't exist or is not a cache model+ bool remove(const std::string & name);// wait until the model instance is fully loaded (thread-safe)+ // note: predicate is called while holding the lock// return when the model no longer in "loading" state- void wait_until_loading_finished(const std::string & name);+ void wait(const std::string & name, std::function<bool(const server_model_meta &)> predicate);+ void wait(std::unique_lock<std::mutex> & lk, const std::string & name, std::function<bool(const server_model_meta &)> predicate);// ensure the model is in ready state (thread-safe)// return false if model is ready@@ -176,8 +206,9 @@ public:struct server_models_routes {common_params params;- json ui_settings = json::object(); // Primary: new name- json webui_settings = json::object(); // Deprecated: use ui_settings (kept for compat)+ json ui_settings = json::object(); // Primary: new name+ json webui_settings = json::object(); // Deprecated: use ui_settings (kept for compat)+ std::atomic<bool> stopping = false; // for graceful disconnecting SSE clients during shutdownserver_models models;server_models_routes(const common_params & params, int argc, char ** argv): params(params), models(params, argc, argv) {@@ -206,6 +237,10 @@ struct server_models_routes {server_http_context::handler_t get_router_models;server_http_context::handler_t post_router_models_load;server_http_context::handler_t post_router_models_unload;+ // management API+ server_http_context::handler_t get_router_models_sse;+ server_http_context::handler_t post_router_models;+ server_http_context::handler_t del_router_models;};/**diff --git a/tools/server/server-queue.cpp b/tools/server/server-queue.cppindex 32cfe7830c..5d37c34536 100644--- a/tools/server/server-queue.cpp+++ b/tools/server/server-queue.cpp@@ -331,6 +331,17 @@ void server_response::send(server_task_result_ptr && result) {}}+void server_response::broadcast(server_task_result_ptr && result) {+ std::unique_lock<std::mutex> lock(mutex_results);+ for (const auto & id_task : waiting_task_ids) {+ RES_DBG("task id = %d pushed to result queue\n", id_task);+ server_task_result_ptr res_copy(result->clone());+ res_copy->id = id_task; // override id with target task id+ queue_results.emplace_back(std::move(res_copy));+ }+ condition_results.notify_all();+}+void server_response::terminate() {running = false;condition_results.notify_all();diff --git a/tools/server/server-queue.h b/tools/server/server-queue.hindex 35f010401f..0b674d6ff0 100644--- a/tools/server/server-queue.h+++ b/tools/server/server-queue.h@@ -154,11 +154,15 @@ public:// Send a new result to a waiting id_taskvoid send(server_task_result_ptr && result);+ // broadcast a new result to all waiting tasks+ // (used by router mode)+ void broadcast(server_task_result_ptr && result);+// terminate the waiting loopvoid terminate();};-// utility class to make working with server_queue and server_response easier+// RAII wrapper to make working with server_queue and server_response easier// it provides a generator-like API for server responses// support pooling connection state and aggregating multiple resultsstruct server_response_reader {diff --git a/tools/server/server-task.h b/tools/server/server-task.hindex bdadcff765..1a03d5f266 100644--- a/tools/server/server-task.h+++ b/tools/server/server-task.h@@ -312,6 +312,9 @@ struct server_task_result {}virtual json to_json() = 0;virtual ~server_task_result() = default;+ virtual server_task_result * clone() const {+ GGML_ABORT("not implemented for this task type");+ }};// using shared_ptr for polymorphism of server_task_result@@ -649,3 +652,12 @@ struct server_prompt_cache {void update();};++// used exclusively by router mode+struct server_task_result_router : server_task_result {+ json data;+ virtual json to_json() override { return data; }+ virtual server_task_result * clone() const override {+ return new server_task_result_router(*this);+ }+};diff --git a/tools/server/server.cpp b/tools/server/server.cppindex da635c6256..0364d7d3b7 100644--- a/tools/server/server.cpp+++ b/tools/server/server.cpp@@ -174,8 +174,11 @@ int llama_server(int argc, char ** argv) {routes.get_props = models_routes->get_router_props;routes.get_models = models_routes->get_router_models;+ ctx_http.post("/models", ex_wrapper(models_routes->post_router_models));ctx_http.post("/models/load", ex_wrapper(models_routes->post_router_models_load));ctx_http.post("/models/unload", ex_wrapper(models_routes->post_router_models_unload));+ ctx_http.get ("/models/sse", ex_wrapper(models_routes->get_router_models_sse));+ ctx_http.del ("/models", ex_wrapper(models_routes->del_router_models));}ctx_http.get ("/health", ex_wrapper(routes.get_health)); // public endpoint (no API key check)@@ -261,6 +264,7 @@ int llama_server(int argc, char ** argv) {clean_up = [&models_routes]() {SRV_INF("%s: cleaning up before exit...\n", __func__);if (models_routes.has_value()) {+ models_routes->stopping.store(true); // maybe redundant, but just to be safemodels_routes->models.unload_all();}llama_backend_free();@@ -274,6 +278,10 @@ int llama_server(int argc, char ** argv) {ctx_http.is_ready.store(true);shutdown_handler = [&](int) {+ if (models_routes.has_value()) {+ // important to disconnect any SSE clients+ models_routes->stopping.store(true);+ }ctx_http.stop();};diff --git a/tools/server/tests/unit/test_router.py b/tools/server/tests/unit/test_router.pyindex c93b92b0b2..11c77ca7aa 100644--- a/tools/server/tests/unit/test_router.py+++ b/tools/server/tests/unit/test_router.py@@ -1,3 +1,4 @@+import threadingimport pytestfrom utils import *@@ -253,3 +254,98 @@ def test_router_reload_models():assert "model-reload-c" in ids, "newly added model should appear"finally:os.remove(preset_path)+++MODEL_DOWNLOAD_ID = "ggml-org/test-model-router-download:F16"+MODEL_DOWNLOAD_TIMEOUT = 300+++def _listen_sse(server: ServerProcess, collected: list, stop: threading.Event):+ """Collect /models/sse events into `collected` until `stop` is set."""+ url = f"http://{server.server_host}:{server.server_port}/models/sse"+ try:+ with requests.get(url, stream=True, timeout=MODEL_DOWNLOAD_TIMEOUT) as resp:+ for line_bytes in resp.iter_lines():+ if stop.is_set():+ break+ line = line_bytes.decode("utf-8")+ if line.startswith("data: "):+ collected.append(json.loads(line[6:]))+ except Exception:+ pass+++def _wait_for_sse_event(collected: list, event_type: str, model: str, timeout: int) -> bool:+ deadline = time.time() + timeout+ while time.time() < deadline:+ if any(e.get("event") == event_type and e.get("model") == model for e in collected):+ return True+ time.sleep(0.5)+ return False+++def test_router_download_model():+ """Case 1: download a model, verify SSE events and GET /models."""+ global server+ server.start()++ # Ensure the model is not present before we start+ server.make_request("DELETE", f"/models?model={MODEL_DOWNLOAD_ID}")++ sse_events: list = []+ stop = threading.Event()+ sse_thread = threading.Thread(+ target=_listen_sse, args=(server, sse_events, stop), daemon=True+ )+ sse_thread.start()++ # Trigger the download+ res = server.make_request("POST", "/models", data={"model": MODEL_DOWNLOAD_ID})+ assert res.status_code == 200+ assert res.body.get("success") is True++ # Wait for download_finished SSE event+ finished = _wait_for_sse_event(+ sse_events, "download_finished", MODEL_DOWNLOAD_ID, MODEL_DOWNLOAD_TIMEOUT+ )+ stop.set()++ assert finished, "Never received download_finished SSE event"+ assert any(+ e.get("event") == "download_progress" and e.get("model") == MODEL_DOWNLOAD_ID+ for e in sse_events+ ), "No download_progress events received"++ # Model should now appear in GET /models+ ids = _get_model_ids(is_reload=False)+ assert MODEL_DOWNLOAD_ID in ids, f"{MODEL_DOWNLOAD_ID} not found in /models after download"+++def test_router_delete_model():+ """Case 2: delete the downloaded model, verify it disappears from GET /models."""+ global server+ server.start()++ # Ensure the model exists (download it if needed)+ if MODEL_DOWNLOAD_ID not in _get_model_ids(is_reload=False):+ res = server.make_request("POST", "/models", data={"model": MODEL_DOWNLOAD_ID})+ assert res.status_code == 200+ sse_events: list = []+ stop = threading.Event()+ threading.Thread(+ target=_listen_sse, args=(server, sse_events, stop), daemon=True+ ).start()+ finished = _wait_for_sse_event(+ sse_events, "download_finished", MODEL_DOWNLOAD_ID, MODEL_DOWNLOAD_TIMEOUT+ )+ stop.set()+ assert finished, "Model did not finish downloading before delete test"++ # Delete the model+ del_res = server.make_request("DELETE", f"/models?model={MODEL_DOWNLOAD_ID}")+ assert del_res.status_code == 200+ assert del_res.body.get("success") is True++ # Model should no longer appear in GET /models+ ids = _get_model_ids(is_reload=False)+ assert MODEL_DOWNLOAD_ID not in ids, f"{MODEL_DOWNLOAD_ID} still present after deletion"diff --git a/tools/server/tests/utils.py b/tools/server/tests/utils.pyindex c5dba1c139..c50c9a0f5a 100644--- a/tools/server/tests/utils.py+++ b/tools/server/tests/utils.py@@ -340,6 +340,9 @@ class ServerProcess:elif method == "POST":response = requests.post(url, headers=headers, json=data, timeout=timeout)parse_body = True+ elif method == "DELETE":+ response = requests.delete(url, headers=headers, timeout=timeout)+ parse_body = Trueelif method == "OPTIONS":response = requests.options(url, headers=headers, timeout=timeout)else: