Skip to content

LLM inference in C/C++

C++ ggml

common: fix explicit -md precedence over draft sidecar resolution (#26165)

Pascalddfc2288e426
diff --git a/common/arg.cpp b/common/arg.cpp
index 84c19e0632..3bc9574d81 100644
--- a/common/arg.cpp
+++ b/common/arg.cpp
@@ -539,6 +539,13 @@ void common_models_handler_apply(common_models_handler & handler, common_params
}
};
+ // an explicit draft file selection (e.g. -md with -hfd) disables the sidecar resolution of the draft repo
+ if (!params.speculative.draft.mparams.hf_file.empty()) {
+ plan_spec.mtp = {};
+ plan_spec.dflash = {};
+ plan_spec.eagle3 = {};
+ }
+
// infer the speculative type from the sidecar shipped by the draft repo when none is requested
if (spec_types_is_default(params)) {
if (!plan_spec.mtp.local_path.empty()) {
@@ -588,6 +595,11 @@ void common_models_handler_apply(common_models_handler & handler, common_params
});
}
+ // a wired draft sidecar counts as an explicit draft for the main plan fallback below
+ if (spec_sidecar_found) {
+ had_spec_url = true;
+ }
+
// handle plan_spec (e.g. --spec-draft-hf)
if (!plan_spec.model_files.empty() && !had_spec_url && !spec_sidecar_found) {
add_tasks(plan_spec.model_files, plan_spec.primary, params.speculative.draft.mparams);
diff --git a/common/download.cpp b/common/download.cpp
index e8e938426f..3776c6c7eb 100644
--- a/common/download.cpp
+++ b/common/download.cpp
@@ -568,16 +568,30 @@ static hf_cache::hf_files get_split_files(const hf_cache::hf_files & files,
}
// pick the best sibling GGUF whose filename contains `keyword` (e.g. "mmproj" / "mtp"),
-// preferring deeper shared directory prefix with the model, then closest quantization
+// preferring deeper shared directory prefix with the model, then exact `tag` match,
+// then closest quantization to the tag when given, or to the model otherwise
static hf_cache::hf_file find_best_sibling(const hf_cache::hf_files & files,
const std::string & model,
- const std::string & keyword) {
+ const std::string & keyword,
+ const std::string & tag = "") {
hf_cache::hf_file best;
size_t best_depth = 0;
int best_diff = 0;
+ bool best_exact = false;
bool found = false;
- auto model_bits = extract_quant_bits(model);
+ std::string tag_upper = tag;
+ for (char & c : tag_upper) {
+ c = (char) std::toupper((unsigned char) c);
+ }
+
+ int model_bits = 0;
+ if (!tag_upper.empty()) {
+ auto pos = tag_upper.find_first_of("0123456789");
+ model_bits = pos == std::string::npos ? 0 : std::stoi(tag_upper.substr(pos));
+ } else {
+ model_bits = extract_quant_bits(model);
+ }
auto model_parts = string_split<std::string>(model, '/');
auto model_dir = model_parts.end() - 1;
@@ -600,10 +614,19 @@ static hf_cache::hf_file find_best_sibling(const hf_cache::hf_files & files,
auto bits = extract_quant_bits(f.path);
auto diff = std::abs(bits - model_bits);
- if (!found || depth > best_depth || (depth == best_depth && diff < best_diff)) {
+ std::string path_upper = f.path;
+ for (char & c : path_upper) {
+ c = (char) std::toupper((unsigned char) c);
+ }
+ bool exact = !tag_upper.empty() && path_upper.find("-" + tag_upper + ".") != std::string::npos;
+
+ if (!found || depth > best_depth ||
+ (depth == best_depth && exact && !best_exact) ||
+ (depth == best_depth && exact == best_exact && diff < best_diff)) {
best = f;
best_depth = depth;
best_diff = diff;
+ best_exact = exact;
found = true;
}
}
@@ -616,18 +639,21 @@ static hf_cache::hf_file find_best_mmproj(const hf_cache::hf_files & files,
}
static hf_cache::hf_file find_best_mtp(const hf_cache::hf_files & files,
- const std::string & model) {
- return find_best_sibling(files, model, "mtp-");
+ const std::string & model,
+ const std::string & tag = "") {
+ return find_best_sibling(files, model, "mtp-", tag);
}
static hf_cache::hf_file find_best_eagle3(const hf_cache::hf_files & files,
- const std::string & model) {
- return find_best_sibling(files, model, "eagle3-");
+ const std::string & model,
+ const std::string & tag = "") {
+ return find_best_sibling(files, model, "eagle3-", tag);
}
static hf_cache::hf_file find_best_dflash(const hf_cache::hf_files & files,
- const std::string & model) {
- return find_best_sibling(files, model, "dflash-");
+ const std::string & model,
+ const std::string & tag = "") {
+ return find_best_sibling(files, model, "dflash-", tag);
}
static bool gguf_filename_is_model(const std::string & filepath) {
@@ -736,27 +762,36 @@ common_download_hf_plan common_download_get_hf_plan(const common_params_model &
}
} else {
primary = find_best_model(all, tag);
- if (primary.path.empty()) {
+ // a requested sidecar can resolve on its own, without a full model of the same tag
+ if (primary.path.empty() && !opts.download_mtp && !opts.download_dflash && !opts.download_eagle3) {
LOG_ERR("%s: no GGUF files found in repository %s\n", __func__, repo.c_str());
list_available_gguf_files(all);
return plan;
}
}
- plan.primary = primary;
- plan.model_files = get_split_files(all, primary);
+ if (!primary.path.empty()) {
+ plan.primary = primary;
+ plan.model_files = get_split_files(all, primary);
+ }
- if (opts.download_mmproj) {
+ if (opts.download_mmproj && !primary.path.empty()) {
plan.mmproj = find_best_mmproj(all, primary.path);
}
if (opts.download_mtp) {
- plan.mtp = find_best_mtp(all, primary.path);
+ plan.mtp = find_best_mtp(all, primary.path, tag);
}
if (opts.download_dflash) {
- plan.dflash = find_best_dflash(all, primary.path);
+ plan.dflash = find_best_dflash(all, primary.path, tag);
}
if (opts.download_eagle3) {
- plan.eagle3 = find_best_eagle3(all, primary.path);
+ plan.eagle3 = find_best_eagle3(all, primary.path, tag);
+ }
+
+ if (primary.path.empty() &&
+ plan.mtp.local_path.empty() && plan.dflash.local_path.empty() && plan.eagle3.local_path.empty()) {
+ LOG_ERR("%s: no GGUF files found in repository %s\n", __func__, repo.c_str());
+ list_available_gguf_files(all);
}
return plan;
diff --git a/common/speculative.cpp b/common/speculative.cpp
index 3cb08767bd..ee94d7c376 100644
--- a/common/speculative.cpp
+++ b/common/speculative.cpp
@@ -2284,7 +2284,7 @@ common_speculative_init_result::common_speculative_init_result(
std::string model_path;
if (has_draft) {
model_path = params.speculative.draft.mparams.path;
- LOG_TRC("%s: loading draft model '%s'\n", __func__, model_path.c_str());
+ LOG_INF("%s: loading draft model '%s'\n", __func__, model_path.c_str());
llama_model * model_dft = llama_model_load_from_file(params.model.path.c_str(), mparams);
if (model_dft == NULL) {
@@ -2304,7 +2304,7 @@ common_speculative_init_result::common_speculative_init_result(
} else if (spec_mtp) {
model_path = params.model.path;
- LOG_TRC("%s: creating MTP draft context against the target model '%s'\n", __func__, model_path.c_str());
+ LOG_INF("%s: creating MTP draft context against the target model '%s'\n", __func__, model_path.c_str());
llama_context * ctx_dft = llama_init_from_model(model_tgt, cparams);
if (ctx_dft == nullptr) {