Skip to content

LLM inference in C/C++

C++ ggml

common: infer the speculative type from the draft repo sidecars (#25989)

Pascal6d5a910c503d
diff --git a/common/arg.cpp b/common/arg.cpp
index 48e2747742..1f5b5a8abf 100644
--- a/common/arg.cpp
+++ b/common/arg.cpp
@@ -351,6 +351,10 @@ static std::string get_default_local_path(const std::string & url) {
return fs_get_cache_file(string_split<std::string>(f, '/').back());
}
+static bool spec_types_is_default(const common_params & params) {
+ return params.speculative.types == std::vector<enum common_speculative_type>{COMMON_SPECULATIVE_TYPE_NONE};
+}
+
common_models_handler common_models_handler_init(const common_params & params, llama_example curr_ex) {
common_download_hf_plan plan;
common_download_hf_plan plan_spec;
@@ -391,7 +395,14 @@ common_models_handler common_models_handler_init(const common_params & params, l
}
if (!params.speculative.draft.mparams.hf_repo.empty()) {
- plan_spec = common_download_get_hf_plan(params.speculative.draft.mparams, opts);
+ // without a requested type, discover every sidecar the draft repo ships to infer the type later
+ auto opts_spec = opts;
+ if (spec_types_is_default(params)) {
+ opts_spec.download_mtp = true;
+ opts_spec.download_dflash = true;
+ opts_spec.download_eagle3 = true;
+ }
+ plan_spec = common_download_get_hf_plan(params.speculative.draft.mparams, opts_spec);
}
if (!params.vocoder.model.hf_repo.empty()) {
@@ -527,6 +538,20 @@ void common_models_handler_apply(common_models_handler & handler, common_params
}
};
+ // infer the speculative type from the sidecar shipped by the draft repo when none is requested
+ if (spec_types_is_default(params)) {
+ if (!plan_spec.mtp.local_path.empty()) {
+ params.speculative.types = { COMMON_SPECULATIVE_TYPE_DRAFT_MTP };
+ plan_spec.dflash = {};
+ plan_spec.eagle3 = {};
+ } else if (!plan_spec.dflash.local_path.empty()) {
+ params.speculative.types = { COMMON_SPECULATIVE_TYPE_DRAFT_DFLASH };
+ plan_spec.eagle3 = {};
+ } else if (!plan_spec.eagle3.local_path.empty()) {
+ params.speculative.types = { COMMON_SPECULATIVE_TYPE_DRAFT_EAGLE3 };
+ }
+ }
+
// when a sidecar type is requested, the draft repo resolves to its sidecar instead of a full model
const bool spec_sidecar_found = !plan_spec.mtp.local_path.empty() ||
!plan_spec.dflash.local_path.empty() ||