LLM inference in C/C++
C++ ggml
common: support the DSpark sidecar resolution (#26458)
Pascalbb4e0e1b3f6b
diff --git a/common/arg.cpp b/common/arg.cppindex 79480e06f9..772422f682 100644--- a/common/arg.cpp+++ b/common/arg.cpp@@ -374,6 +374,10 @@ common_models_handler common_models_handler_init(const common_params & params, lparams.speculative.types.end(),COMMON_SPECULATIVE_TYPE_DRAFT_EAGLE3) != params.speculative.types.end();+ const bool spec_type_draft_dspark = std::find(params.speculative.types.begin(),+ params.speculative.types.end(),+ COMMON_SPECULATIVE_TYPE_DRAFT_DSPARK) != params.speculative.types.end();+// only download mmproj if the current example is using itbool use_mmproj = false;for (const auto & ex : mmproj_examples) {@@ -388,6 +392,7 @@ common_models_handler common_models_handler_init(const common_params & params, lopts.download_mtp = spec_type_draft_mtp;opts.download_eagle3 = spec_type_draft_eagle3;opts.download_dflash = spec_type_draft_dflash;+ opts.download_dspark = spec_type_draft_dspark;opts.download_mmproj = use_mmproj && !params.no_mmproj&& params.mmproj.path.empty() && params.mmproj.url.empty();@@ -402,6 +407,7 @@ common_models_handler common_models_handler_init(const common_params & params, lopts_spec.download_mtp = true;opts_spec.download_dflash = true;opts_spec.download_eagle3 = true;+ opts_spec.download_dspark = true;}plan_spec = common_download_get_hf_plan(params.speculative.draft.mparams, opts_spec);}@@ -544,12 +550,19 @@ void common_models_handler_apply(common_models_handler & handler, common_paramsplan_spec.mtp = {};plan_spec.dflash = {};plan_spec.eagle3 = {};+ plan_spec.dspark = {};}// infer the speculative type from the sidecar shipped by the draft repo when none is requestedif (spec_types_is_default(params)) {if (!plan_spec.mtp.local_path.empty()) {params.speculative.types = { COMMON_SPECULATIVE_TYPE_DRAFT_MTP };+ plan_spec.dspark = {};+ plan_spec.dflash = {};+ plan_spec.eagle3 = {};+ } else if (!plan_spec.dspark.local_path.empty()) {+ // dspark outranks dflash, its sidecar carries the extra Markov head+ params.speculative.types = { COMMON_SPECULATIVE_TYPE_DRAFT_DSPARK };plan_spec.dflash = {};plan_spec.eagle3 = {};} else if (!plan_spec.dflash.local_path.empty()) {@@ -563,7 +576,8 @@ void common_models_handler_apply(common_models_handler & handler, common_params// when a sidecar type is requested, the draft repo resolves to its sidecar instead of a full modelconst bool spec_sidecar_found = !plan_spec.mtp.local_path.empty() ||!plan_spec.dflash.local_path.empty() ||- !plan_spec.eagle3.local_path.empty();+ !plan_spec.eagle3.local_path.empty() ||+ !plan_spec.dspark.local_path.empty();if (!plan_spec.mtp.local_path.empty() && !had_spec_url) {tasks.emplace_back(plan_spec.mtp, opts, [&]() {// only use the discovered MTP head when no draft path is set yet@@ -594,6 +608,16 @@ void common_models_handler_apply(common_models_handler & handler, common_params}});}+ if (!plan_spec.dspark.local_path.empty() && !had_spec_url) {+ tasks.emplace_back(plan_spec.dspark, opts, [&]() {+ // only use the discovered DSpark sidecar when no draft path is set yet+ if (params.speculative.draft.mparams.path.empty()) {+ params.speculative.draft.mparams.path = hf_cache::finalize_file(plan_spec.dspark);+ } else {+ hf_cache::finalize_file(plan_spec.dspark);+ }+ });+ }// a wired draft sidecar counts as an explicit draft for the main plan fallback belowif (spec_sidecar_found) {@@ -649,6 +673,16 @@ void common_models_handler_apply(common_models_handler & handler, common_params}});}+ if (!plan.dspark.local_path.empty() && !had_spec_url) {+ tasks.emplace_back(plan.dspark, opts, [&]() {+ // only fall back to the discovered DSpark sidecar when no draft was explicitly provided+ if (params.speculative.draft.mparams.empty()) {+ params.speculative.draft.mparams.path = hf_cache::finalize_file(plan.dspark);+ } else {+ hf_cache::finalize_file(plan.dspark);+ }+ });+ }if (!plan.preset.local_path.empty()) {tasks.emplace_back(plan.preset, opts, [&]() {// if HF repo is a preset repo, we simply run server in router mode with the preset.ini filediff --git a/common/download.cpp b/common/download.cppindex 3776c6c7eb..44c6cea424 100644--- a/common/download.cpp+++ b/common/download.cpp@@ -656,6 +656,12 @@ static hf_cache::hf_file find_best_dflash(const hf_cache::hf_files & files,return find_best_sibling(files, model, "dflash-", tag);}+static hf_cache::hf_file find_best_dspark(const hf_cache::hf_files & files,+ const std::string & model,+ const std::string & tag = "") {+ return find_best_sibling(files, model, "dspark-", tag);+}+static bool gguf_filename_is_model(const std::string & filepath) {if (!string_ends_with(filepath, ".gguf")) {return false;@@ -670,7 +676,8 @@ static bool gguf_filename_is_model(const std::string & filepath) {filename.find("imatrix") == std::string::npos &&filename.find("mtp-") == std::string::npos &&filename.find("eagle3-") == std::string::npos &&- filename.find("dflash-") == std::string::npos;+ filename.find("dflash-") == std::string::npos &&+ filename.find("dspark-") == std::string::npos;}static hf_cache::hf_file find_best_model(const hf_cache::hf_files & files,@@ -763,7 +770,7 @@ common_download_hf_plan common_download_get_hf_plan(const common_params_model &} else {primary = find_best_model(all, tag);// a requested sidecar can resolve on its own, without a full model of the same tag- if (primary.path.empty() && !opts.download_mtp && !opts.download_dflash && !opts.download_eagle3) {+ if (primary.path.empty() && !opts.download_mtp && !opts.download_dflash && !opts.download_eagle3 && !opts.download_dspark) {LOG_ERR("%s: no GGUF files found in repository %s\n", __func__, repo.c_str());list_available_gguf_files(all);return plan;@@ -787,9 +794,12 @@ common_download_hf_plan common_download_get_hf_plan(const common_params_model &if (opts.download_eagle3) {plan.eagle3 = find_best_eagle3(all, primary.path, tag);}+ if (opts.download_dspark) {+ plan.dspark = find_best_dspark(all, primary.path, tag);+ }if (primary.path.empty() &&- plan.mtp.local_path.empty() && plan.dflash.local_path.empty() && plan.eagle3.local_path.empty()) {+ plan.mtp.local_path.empty() && plan.dflash.local_path.empty() && plan.eagle3.local_path.empty() && plan.dspark.local_path.empty()) {LOG_ERR("%s: no GGUF files found in repository %s\n", __func__, repo.c_str());list_available_gguf_files(all);}@@ -967,7 +977,8 @@ std::vector<common_cached_model_info> common_list_cached_models() {split.prefix.find("mmproj") != std::string::npos ||split.prefix.find("mtp-") != std::string::npos ||split.prefix.find("eagle3-") != std::string::npos ||- split.prefix.find("dflash-") != std::string::npos) {+ split.prefix.find("dflash-") != std::string::npos ||+ split.prefix.find("dspark-") != std::string::npos) {continue;}if (seen.insert(f.repo_id + ":" + split.tag).second) {diff --git a/common/download.h b/common/download.hindex 3e789e9e93..9a03f5e914 100644--- a/common/download.h+++ b/common/download.h@@ -59,6 +59,7 @@ struct common_download_opts {bool download_mtp = false;bool download_eagle3 = false;bool download_dflash = false;+ bool download_dspark = false;common_download_callback * callback = nullptr;};@@ -110,6 +111,7 @@ struct common_download_hf_plan {hf_cache::hf_file mtp;hf_cache::hf_file eagle3;hf_cache::hf_file dflash;+ hf_cache::hf_file dspark;hf_cache::hf_file preset; // if set, only this file is downloaded};common_download_hf_plan common_download_get_hf_plan(const common_params_model & model, const common_download_opts & opts);