LLM inference in C/C++
C++ ggml
common: infer the speculative type from the draft repo sidecars (#25989)
Pascal6d5a910c503d
diff --git a/common/arg.cpp b/common/arg.cppindex 48e2747742..1f5b5a8abf 100644--- a/common/arg.cpp+++ b/common/arg.cpp@@ -351,6 +351,10 @@ static std::string get_default_local_path(const std::string & url) {return fs_get_cache_file(string_split<std::string>(f, '/').back());}+static bool spec_types_is_default(const common_params & params) {+ return params.speculative.types == std::vector<enum common_speculative_type>{COMMON_SPECULATIVE_TYPE_NONE};+}+common_models_handler common_models_handler_init(const common_params & params, llama_example curr_ex) {common_download_hf_plan plan;common_download_hf_plan plan_spec;@@ -391,7 +395,14 @@ common_models_handler common_models_handler_init(const common_params & params, l}if (!params.speculative.draft.mparams.hf_repo.empty()) {- plan_spec = common_download_get_hf_plan(params.speculative.draft.mparams, opts);+ // without a requested type, discover every sidecar the draft repo ships to infer the type later+ auto opts_spec = opts;+ if (spec_types_is_default(params)) {+ opts_spec.download_mtp = true;+ opts_spec.download_dflash = true;+ opts_spec.download_eagle3 = true;+ }+ plan_spec = common_download_get_hf_plan(params.speculative.draft.mparams, opts_spec);}if (!params.vocoder.model.hf_repo.empty()) {@@ -527,6 +538,20 @@ void common_models_handler_apply(common_models_handler & handler, common_params}};+ // infer the speculative type from the sidecar shipped by the draft repo when none is requested+ if (spec_types_is_default(params)) {+ if (!plan_spec.mtp.local_path.empty()) {+ params.speculative.types = { COMMON_SPECULATIVE_TYPE_DRAFT_MTP };+ plan_spec.dflash = {};+ plan_spec.eagle3 = {};+ } else if (!plan_spec.dflash.local_path.empty()) {+ params.speculative.types = { COMMON_SPECULATIVE_TYPE_DRAFT_DFLASH };+ plan_spec.eagle3 = {};+ } else if (!plan_spec.eagle3.local_path.empty()) {+ params.speculative.types = { COMMON_SPECULATIVE_TYPE_DRAFT_EAGLE3 };+ }+ }+// when a sidecar type is requested, the draft repo resolves to its sidecar instead of a full modelconst bool spec_sidecar_found = !plan_spec.mtp.local_path.empty() ||!plan_spec.dflash.local_path.empty() ||