LLM inference in C/C++
C++ ggml
server: add dedup-cache-models preset option (#27346)
Xuan-Son Nguyenee4c505a4fb3
diff --git a/common/arg.cpp b/common/arg.cppindex 0766087c38..6f5fe377d5 100644--- a/common/arg.cpp+++ b/common/arg.cpp@@ -4658,6 +4658,12 @@ void common_params_add_preset_options(std::vector<common_arg> & args) {[](common_params &, int) { /* unused */ }).set_env(COMMON_ARG_PRESET_STOP_TIMEOUT).set_preset_only());+ args.push_back(common_arg(+ {"dedup-cache-models"}, "0|1",+ "in server router mode, hide a cached model from the model list when this preset resolves to the same model file",+ [](common_params &, const std::string &) { /* unused */ }+ ).set_env(COMMON_ARG_PRESET_DEDUP_CACHE_MODELS).set_preset_only());+// args.push_back(common_arg(// {"pin"},// "in server router mode, do not unload this model if models_max is exceeded",diff --git a/common/arg.h b/common/arg.hindex 44b9e887cf..421bc295fc 100644--- a/common/arg.h+++ b/common/arg.h@@ -11,8 +11,9 @@#include <memory>// pseudo-env variable to identify preset-only arguments-#define COMMON_ARG_PRESET_LOAD_ON_STARTUP "__PRESET_LOAD_ON_STARTUP"-#define COMMON_ARG_PRESET_STOP_TIMEOUT "__PRESET_STOP_TIMEOUT"+#define COMMON_ARG_PRESET_LOAD_ON_STARTUP "__PRESET_LOAD_ON_STARTUP"+#define COMMON_ARG_PRESET_STOP_TIMEOUT "__PRESET_STOP_TIMEOUT"+#define COMMON_ARG_PRESET_DEDUP_CACHE_MODELS "__PRESET_DEDUP_CACHE_MODELS"//// CLI argument parsingdiff --git a/common/download.cpp b/common/download.cppindex 44c6cea424..2509f75ab9 100644--- a/common/download.cpp+++ b/common/download.cpp@@ -989,6 +989,26 @@ std::vector<common_cached_model_info> common_list_cached_models() {return result;}+std::string common_download_resolve_path(const std::string & hf_repo_with_tag, const std::string & hf_file) {+ auto [repo, tag] = common_download_split_repo_tag(hf_repo_with_tag);++ auto files = hf_cache::get_cached_files(repo);+ if (files.empty()) {+ return "";+ }++ if (!hf_file.empty()) {+ for (const auto & f : files) {+ if (f.path == hf_file) {+ return f.local_path;+ }+ }+ return "";+ }++ return find_best_model(files, tag).local_path;+}+bool common_download_remove(const std::string & hf_repo_with_tag) {namespace fs = std::filesystem;diff --git a/common/download.h b/common/download.hindex 9a03f5e914..8c30cfc3ea 100644--- a/common/download.h+++ b/common/download.h@@ -85,6 +85,10 @@ std::vector<std::string> common_download_get_all_parts(const std::string & url);// returns list of cached modelsstd::vector<common_cached_model_info> common_list_cached_models();+// resolve the local cached file path for a HF repo without network access (hf_file, if given, must match exactly)+// returns an empty string if the model is not present in the cache+std::string common_download_resolve_path(const std::string & hf_repo_with_tag, const std::string & hf_file = "");+// download single file from url to local path// returns status code or -1 on error// skip_etag: if true, don't read/write .etag files (for HF cache where filename is the hash)diff --git a/tools/server/README.md b/tools/server/README.mdindex 67e52b1db8..78274967db 100644--- a/tools/server/README.md+++ b/tools/server/README.md@@ -1759,6 +1759,7 @@ The precedence rule for preset options is as follows:We also offer additional options that are exclusive to presets (these aren't treated as command-line arguments):- `load-on-startup` (boolean): Controls whether the model loads automatically when the server starts- `stop-timeout` (int, seconds): After requested unload, wait for this many seconds before forcing termination (default: 10)+- `dedup-cache-models` (boolean): When the preset uses `hf-repo` pointing to a model that is already downloaded, hide the corresponding cached model entry from `GET /models` (the preset entry remains visible). Set it in the `[*]` section to apply to all presets.### Routing requestsdiff --git a/tools/server/server-models.cpp b/tools/server/server-models.cppindex df34adc295..35b9355700 100644--- a/tools/server/server-models.cpp+++ b/tools/server/server-models.cpp@@ -555,6 +555,40 @@ void server_models::load_models() {return source_map.count(name) ? source_map.at(name) : SERVER_MODEL_SOURCE_PRESET;};+ // hide cache models whose resolved file is already used by a preset with dedup-cache-models enabled+ std::set<std::string> hidden_models;+ {+ std::set<std::string> preset_paths;+ for (const auto & [name, preset] : custom_presets) {+ std::string val;+ if (!preset.get_option(COMMON_ARG_PRESET_DEDUP_CACHE_MODELS, val) || !common_arg_utils::is_truthy(val)) {+ continue;+ }+ std::string hf_repo;+ if (!preset.get_option("LLAMA_ARG_HF_REPO", hf_repo) || hf_repo.empty()) {+ continue;+ }+ std::string hf_file;+ preset.get_option("LLAMA_ARG_HF_FILE", hf_file);+ std::string path = common_download_resolve_path(hf_repo, hf_file);+ if (!path.empty()) {+ preset_paths.insert(path);+ }+ }+ if (!preset_paths.empty()) {+ for (const auto & [name, preset] : cached_models) {+ if (get_source(name) != SERVER_MODEL_SOURCE_CACHE) {+ continue; // merged with another source, not a pure cache entry+ }+ std::string path = common_download_resolve_path(name);+ if (!path.empty() && preset_paths.count(path)) {+ SRV_INF("hiding cache model name=%s (deduplicated by a preset)\n", name.c_str());+ hidden_models.insert(name);+ }+ }+ }+ }+// Helpers that read `mapping` - must be called while holding the lock.std::unordered_set<std::string> custom_names;for (const auto & [name, preset] : custom_presets) custom_names.insert(name);@@ -590,6 +624,11 @@ void server_models::load_models() {}}};+ auto apply_hidden = [&]() {+ for (auto & [name, inst] : mapping) {+ inst.meta.hidden = hidden_models.count(name) > 0;+ }+ };// update_args() injects HOST/PORT/ALIAS, so strip them before comparing presetsauto preset_options_for_compare = [](common_preset p) {p.unset_option("LLAMA_ARG_HOST");@@ -630,6 +669,7 @@ void server_models::load_models() {add_model(std::move(meta));}apply_stop_timeout();+ apply_hidden();log_available_models();std::vector<std::string> models_to_load;@@ -806,6 +846,7 @@ void server_models::load_models() {}apply_stop_timeout();+ apply_hidden();// clear reload flag before unlocking for autoload - load() blocks on !is_reloading,// so clearing it here (while still locked) prevents a deadlock in the autoload calls below@@ -1929,6 +1970,9 @@ void server_models_routes::init_routes() {auto all_models = models.get_all_meta();std::time_t t = std::time(0);for (const auto & meta : all_models) {+ if (meta.hidden) {+ continue; // cache model deduplicated by a preset+ }json status {{"value", server_model_status_to_string(meta.status)},{"args", meta.args},diff --git a/tools/server/server-models.h b/tools/server/server-models.hindex 615acb577b..79b231cba8 100644--- a/tools/server/server-models.h+++ b/tools/server/server-models.h@@ -84,6 +84,7 @@ struct server_model_meta {int exit_code = 0; // exit code of the model instance process (only valid if status == FAILED)int stop_timeout = 0; // seconds to wait before force-killing the model instance during shutdownmtmd_caps multimodal; // multimodal capabilities+ bool hidden = false; // hidden from GET /models, but still accept if requestedbool is_ready() const {return status == SERVER_MODEL_STATUS_LOADED;diff --git a/tools/server/tests/unit/test_router.py b/tools/server/tests/unit/test_router.pyindex 5ab62666ce..0e1467de37 100644--- a/tools/server/tests/unit/test_router.py+++ b/tools/server/tests/unit/test_router.py@@ -406,6 +406,59 @@ def test_router_reload_models():os.remove(preset_path)+def test_router_dedup_cache_models():+ """dedup-cache-models hides the cache entry backing a preset from GET /models"""+ global server++ preset_path = os.path.join(TMP_DIR, "test_dedup.ini")+ cache_id = "ggml-org/test-model-stories260K:F32"++ with open(preset_path, "w") as f:+ f.write(+ "[model-dedup]\n"+ "hf-repo = ggml-org/test-model-stories260K\n"+ "dedup-cache-models = 1\n"+ )++ server.models_preset = preset_path+ server.start()++ try:+ ids = _get_model_ids(is_reload=False)+ assert "model-dedup" in ids+ assert cache_id not in ids, "cache model should be hidden by dedup"+ # other cache models are unaffected+ assert "ggml-org/tinygemma3-GGUF:Q8_0" in ids++ # the hidden model is only hidden from the listing, it can still be used+ res = server.make_request("POST", "/tokenize", data={"model": cache_id, "content": "hello"})+ assert res.status_code == 200++ # disabling the flag brings the cache entry back on reload+ with open(preset_path, "w") as f:+ f.write(+ "[model-dedup]\n"+ "hf-repo = ggml-org/test-model-stories260K\n"+ )+ ids = _get_model_ids(is_reload=True)+ assert cache_id in ids++ # the flag also works from the global section+ with open(preset_path, "w") as f:+ f.write(+ "[*]\n"+ "dedup-cache-models = 1\n"+ "\n"+ "[model-dedup]\n"+ "hf-repo = ggml-org/test-model-stories260K\n"+ )+ ids = _get_model_ids(is_reload=True)+ assert "model-dedup" in ids+ assert cache_id not in ids, "cache model should be hidden by global dedup"+ finally:+ os.remove(preset_path)++def test_router_remote_preset():global serverserver.model_hf_repo = "ggml-org/test-preset-ci"