LLM inference in C/C++
C++ ggml
common : auto-detect spec type from draft GGUF metadata (#26814)
aic0d3rf65e568fd837
diff --git a/common/arg.cpp b/common/arg.cppindex 9fe7d66e24..1f70d0ad45 100644--- a/common/arg.cpp+++ b/common/arg.cpp@@ -561,6 +561,15 @@ void common_models_handler_apply(common_models_handler & handler, common_params}}+ // infer the speculative type from the draft GGUF metadata when none is requested+ // note: reads only the first split - sharded drafts need an explicit --spec-type+ if (spec_types_is_default(params) && !params.speculative.draft.mparams.path.empty()) {+ const auto types_gguf = common_speculative_types_from_gguf(params.speculative.draft.mparams.path);+ if (!types_gguf.empty()) {+ params.speculative.types = types_gguf;+ }+ }+// when a sidecar type is requested, the draft repo resolves to its sidecar instead of a full modelconst bool spec_sidecar_found = !plan_spec.mtp.local_path.empty() ||!plan_spec.dflash.local_path.empty() ||diff --git a/common/speculative.cpp b/common/speculative.cppindex 652ed4d3a1..862c4f045c 100644--- a/common/speculative.cpp+++ b/common/speculative.cpp@@ -2,6 +2,7 @@#include "common.h"#include "ggml.h"+#include "ggml-cpp.h"#include "llama.h"#include "log.h"#include "ngram-cache.h"@@ -2227,6 +2228,37 @@ common_speculative_type common_speculative_type_from_name(const std::string & nareturn it->second;}+std::vector<common_speculative_type> common_speculative_types_from_gguf(const std::string & path) {+ struct gguf_init_params gguf_params = {+ /* .no_alloc = */ true,+ /* .ctx = */ nullptr,+ };++ gguf_context_ptr gguf_ctx(gguf_init_from_file(path.c_str(), gguf_params));+ if (!gguf_ctx) {+ return {};+ }++ const int64_t arch_id = gguf_find_key(gguf_ctx.get(), "general.architecture");+ if (arch_id < 0 || gguf_get_kv_type(gguf_ctx.get(), arch_id) != GGUF_TYPE_STRING) {+ return {};+ }++ const std::string arch = gguf_get_val_str(gguf_ctx.get(), arch_id);+ if (arch != "dflash") {+ return {};+ }++ // the Markov head distinguishes draft-dspark from draft-dflash+ const auto type = gguf_find_tensor(gguf_ctx.get(), "markov_w1.weight") >= 0+ ? COMMON_SPECULATIVE_TYPE_DRAFT_DSPARK+ : COMMON_SPECULATIVE_TYPE_DRAFT_DFLASH;++ SPC_INF("auto-detected speculative type '%s' from the draft model metadata\n", common_speculative_type_to_str(type).c_str());++ return { type };+}+static uint32_t common_get_enabled_speculative_configs(const std::vector<common_speculative_type> & configs) {uint32_t result = 0;for (size_t i = 0; i < configs.size(); i++) {diff --git a/common/speculative.h b/common/speculative.hindex 06b0992ed6..12ae31b7de 100644--- a/common/speculative.h+++ b/common/speculative.h@@ -14,6 +14,9 @@ const char * common_speculative_all_types_str();// parse user provided typesstd::vector<enum common_speculative_type> common_speculative_types_from_names(const std::vector<std::string> & names);+// infer the spec types from the GGUF metadata of a draft model; empty if unknown+std::vector<enum common_speculative_type> common_speculative_types_from_gguf(const std::string & path);+// convert string to typeenum common_speculative_type common_speculative_type_from_name(const std::string & name);