LLM inference in C/C++
C++ ggml
common: support --models-dir loading MTP assistant models (#24431)
Eric Zhang5f754ea0e2fd
diff --git a/common/preset.cpp b/common/preset.cppindex 0b29af8834..4c61e93eea 100644--- a/common/preset.cpp+++ b/common/preset.cpp@@ -365,8 +365,25 @@ struct local_model {std::string name;std::string path;std::string path_mmproj;+ std::string path_draft;};+// TODO @ngxson: handle "eagle3-" when it's supported by common_speculative_types_from_gguf()+static const char * draft_prefixes[] = { "mtp-", "dspark-", "dflash-" };++static bool is_mmproj_file(const std::string & fname) {+ return fname.find("mmproj") != std::string::npos;+}++static bool is_draft_file(const std::string & fname) {+ for (const auto & prefix : draft_prefixes) {+ if (fname.rfind(prefix, 0) == 0) {+ return true;+ }+ }+ return false;+}+common_presets common_preset_context::load_from_models_dir(const std::string & models_dir) const {if (!std::filesystem::exists(models_dir) || !std::filesystem::is_directory(models_dir)) {throw std::runtime_error(string_format("error: '%s' does not exist or is not a directory\n", models_dir.c_str()));@@ -378,10 +395,15 @@ common_presets common_preset_context::load_from_models_dir(const std::string & mcommon_file_info model_file;common_file_info first_shard_file;common_file_info mmproj_file;+ common_file_info draft_file;for (const auto & file : files) {if (string_ends_with(file.name, ".gguf")) {- if (file.name.find("mmproj") != std::string::npos) {+ if (is_mmproj_file(file.name)) {mmproj_file = file;+ } else if (is_draft_file(file.name)) {+ if (draft_file.path.empty()) {+ draft_file = file; // first sidecar found wins+ }} else if (file.name.find("-00001-of-") != std::string::npos) {first_shard_file = file;} else {@@ -393,7 +415,8 @@ common_presets common_preset_context::load_from_models_dir(const std::string & mlocal_model model{/* name */ name,/* path */ first_shard_file.path.empty() ? model_file.path : first_shard_file.path,- /* path_mmproj */ mmproj_file.path // can be empty+ /* path_mmproj */ mmproj_file.path, // can be empty+ /* path_draft */ draft_file.path // can be empty};if (!model.path.empty()) {models.push_back(model);@@ -405,13 +428,17 @@ common_presets common_preset_context::load_from_models_dir(const std::string & mif (file.is_dir) {scan_subdir(file.path, file.name);} else if (string_ends_with(file.name, ".gguf")) {+ if (is_mmproj_file(file.name) || is_draft_file(file.name)) {+ continue; // companion file, cannot be loaded as a model on its own+ }// single file modelstd::string name = file.name;string_replace_all(name, ".gguf", "");local_model model{/* name */ name,/* path */ file.path,- /* path_mmproj */ ""+ /* path_mmproj */ "",+ /* path_draft */ ""};models.push_back(model);}@@ -426,6 +453,9 @@ common_presets common_preset_context::load_from_models_dir(const std::string & mif (!model.path_mmproj.empty()) {preset.set_option(*this, "LLAMA_ARG_MMPROJ", model.path_mmproj);}+ if (!model.path_draft.empty()) {+ preset.set_option(*this, "LLAMA_ARG_SPEC_DRAFT_MODEL", model.path_draft);+ }out[preset.name] = preset;}