LLM inference in C/C++
C++ ggml
common: fix explicit -md precedence over draft sidecar resolution (#26165)
Pascalddfc2288e426
diff --git a/common/arg.cpp b/common/arg.cppindex 84c19e0632..3bc9574d81 100644--- a/common/arg.cpp+++ b/common/arg.cpp@@ -539,6 +539,13 @@ void common_models_handler_apply(common_models_handler & handler, common_params}};+ // an explicit draft file selection (e.g. -md with -hfd) disables the sidecar resolution of the draft repo+ if (!params.speculative.draft.mparams.hf_file.empty()) {+ plan_spec.mtp = {};+ plan_spec.dflash = {};+ plan_spec.eagle3 = {};+ }+// infer the speculative type from the sidecar shipped by the draft repo when none is requestedif (spec_types_is_default(params)) {if (!plan_spec.mtp.local_path.empty()) {@@ -588,6 +595,11 @@ void common_models_handler_apply(common_models_handler & handler, common_params});}+ // a wired draft sidecar counts as an explicit draft for the main plan fallback below+ if (spec_sidecar_found) {+ had_spec_url = true;+ }+// handle plan_spec (e.g. --spec-draft-hf)if (!plan_spec.model_files.empty() && !had_spec_url && !spec_sidecar_found) {add_tasks(plan_spec.model_files, plan_spec.primary, params.speculative.draft.mparams);diff --git a/common/download.cpp b/common/download.cppindex e8e938426f..3776c6c7eb 100644--- a/common/download.cpp+++ b/common/download.cpp@@ -568,16 +568,30 @@ static hf_cache::hf_files get_split_files(const hf_cache::hf_files & files,}// pick the best sibling GGUF whose filename contains `keyword` (e.g. "mmproj" / "mtp"),-// preferring deeper shared directory prefix with the model, then closest quantization+// preferring deeper shared directory prefix with the model, then exact `tag` match,+// then closest quantization to the tag when given, or to the model otherwisestatic hf_cache::hf_file find_best_sibling(const hf_cache::hf_files & files,const std::string & model,- const std::string & keyword) {+ const std::string & keyword,+ const std::string & tag = "") {hf_cache::hf_file best;size_t best_depth = 0;int best_diff = 0;+ bool best_exact = false;bool found = false;- auto model_bits = extract_quant_bits(model);+ std::string tag_upper = tag;+ for (char & c : tag_upper) {+ c = (char) std::toupper((unsigned char) c);+ }++ int model_bits = 0;+ if (!tag_upper.empty()) {+ auto pos = tag_upper.find_first_of("0123456789");+ model_bits = pos == std::string::npos ? 0 : std::stoi(tag_upper.substr(pos));+ } else {+ model_bits = extract_quant_bits(model);+ }auto model_parts = string_split<std::string>(model, '/');auto model_dir = model_parts.end() - 1;@@ -600,10 +614,19 @@ static hf_cache::hf_file find_best_sibling(const hf_cache::hf_files & files,auto bits = extract_quant_bits(f.path);auto diff = std::abs(bits - model_bits);- if (!found || depth > best_depth || (depth == best_depth && diff < best_diff)) {+ std::string path_upper = f.path;+ for (char & c : path_upper) {+ c = (char) std::toupper((unsigned char) c);+ }+ bool exact = !tag_upper.empty() && path_upper.find("-" + tag_upper + ".") != std::string::npos;++ if (!found || depth > best_depth ||+ (depth == best_depth && exact && !best_exact) ||+ (depth == best_depth && exact == best_exact && diff < best_diff)) {best = f;best_depth = depth;best_diff = diff;+ best_exact = exact;found = true;}}@@ -616,18 +639,21 @@ static hf_cache::hf_file find_best_mmproj(const hf_cache::hf_files & files,}static hf_cache::hf_file find_best_mtp(const hf_cache::hf_files & files,- const std::string & model) {- return find_best_sibling(files, model, "mtp-");+ const std::string & model,+ const std::string & tag = "") {+ return find_best_sibling(files, model, "mtp-", tag);}static hf_cache::hf_file find_best_eagle3(const hf_cache::hf_files & files,- const std::string & model) {- return find_best_sibling(files, model, "eagle3-");+ const std::string & model,+ const std::string & tag = "") {+ return find_best_sibling(files, model, "eagle3-", tag);}static hf_cache::hf_file find_best_dflash(const hf_cache::hf_files & files,- const std::string & model) {- return find_best_sibling(files, model, "dflash-");+ const std::string & model,+ const std::string & tag = "") {+ return find_best_sibling(files, model, "dflash-", tag);}static bool gguf_filename_is_model(const std::string & filepath) {@@ -736,27 +762,36 @@ common_download_hf_plan common_download_get_hf_plan(const common_params_model &}} else {primary = find_best_model(all, tag);- if (primary.path.empty()) {+ // a requested sidecar can resolve on its own, without a full model of the same tag+ if (primary.path.empty() && !opts.download_mtp && !opts.download_dflash && !opts.download_eagle3) {LOG_ERR("%s: no GGUF files found in repository %s\n", __func__, repo.c_str());list_available_gguf_files(all);return plan;}}- plan.primary = primary;- plan.model_files = get_split_files(all, primary);+ if (!primary.path.empty()) {+ plan.primary = primary;+ plan.model_files = get_split_files(all, primary);+ }- if (opts.download_mmproj) {+ if (opts.download_mmproj && !primary.path.empty()) {plan.mmproj = find_best_mmproj(all, primary.path);}if (opts.download_mtp) {- plan.mtp = find_best_mtp(all, primary.path);+ plan.mtp = find_best_mtp(all, primary.path, tag);}if (opts.download_dflash) {- plan.dflash = find_best_dflash(all, primary.path);+ plan.dflash = find_best_dflash(all, primary.path, tag);}if (opts.download_eagle3) {- plan.eagle3 = find_best_eagle3(all, primary.path);+ plan.eagle3 = find_best_eagle3(all, primary.path, tag);+ }++ if (primary.path.empty() &&+ plan.mtp.local_path.empty() && plan.dflash.local_path.empty() && plan.eagle3.local_path.empty()) {+ LOG_ERR("%s: no GGUF files found in repository %s\n", __func__, repo.c_str());+ list_available_gguf_files(all);}return plan;diff --git a/common/speculative.cpp b/common/speculative.cppindex 3cb08767bd..ee94d7c376 100644--- a/common/speculative.cpp+++ b/common/speculative.cpp@@ -2284,7 +2284,7 @@ common_speculative_init_result::common_speculative_init_result(std::string model_path;if (has_draft) {model_path = params.speculative.draft.mparams.path;- LOG_TRC("%s: loading draft model '%s'\n", __func__, model_path.c_str());+ LOG_INF("%s: loading draft model '%s'\n", __func__, model_path.c_str());llama_model * model_dft = llama_model_load_from_file(params.model.path.c_str(), mparams);if (model_dft == NULL) {@@ -2304,7 +2304,7 @@ common_speculative_init_result::common_speculative_init_result(} else if (spec_mtp) {model_path = params.model.path;- LOG_TRC("%s: creating MTP draft context against the target model '%s'\n", __func__, model_path.c_str());+ LOG_INF("%s: creating MTP draft context against the target model '%s'\n", __func__, model_path.c_str());llama_context * ctx_dft = llama_init_from_model(model_tgt, cparams);if (ctx_dft == nullptr) {