Skip to content

LLM inference in C/C++

C++ ggml

common : auto-detect spec type from draft GGUF metadata (#26814)

aic0d3rf65e568fd837
diff --git a/common/arg.cpp b/common/arg.cpp
index 9fe7d66e24..1f70d0ad45 100644
--- a/common/arg.cpp
+++ b/common/arg.cpp
@@ -561,6 +561,15 @@ void common_models_handler_apply(common_models_handler & handler, common_params
}
}
+ // infer the speculative type from the draft GGUF metadata when none is requested
+ // note: reads only the first split - sharded drafts need an explicit --spec-type
+ if (spec_types_is_default(params) && !params.speculative.draft.mparams.path.empty()) {
+ const auto types_gguf = common_speculative_types_from_gguf(params.speculative.draft.mparams.path);
+ if (!types_gguf.empty()) {
+ params.speculative.types = types_gguf;
+ }
+ }
+
// when a sidecar type is requested, the draft repo resolves to its sidecar instead of a full model
const bool spec_sidecar_found = !plan_spec.mtp.local_path.empty() ||
!plan_spec.dflash.local_path.empty() ||
diff --git a/common/speculative.cpp b/common/speculative.cpp
index 652ed4d3a1..862c4f045c 100644
--- a/common/speculative.cpp
+++ b/common/speculative.cpp
@@ -2,6 +2,7 @@
#include "common.h"
#include "ggml.h"
+#include "ggml-cpp.h"
#include "llama.h"
#include "log.h"
#include "ngram-cache.h"
@@ -2227,6 +2228,37 @@ common_speculative_type common_speculative_type_from_name(const std::string & na
return it->second;
}
+std::vector<common_speculative_type> common_speculative_types_from_gguf(const std::string & path) {
+ struct gguf_init_params gguf_params = {
+ /* .no_alloc = */ true,
+ /* .ctx = */ nullptr,
+ };
+
+ gguf_context_ptr gguf_ctx(gguf_init_from_file(path.c_str(), gguf_params));
+ if (!gguf_ctx) {
+ return {};
+ }
+
+ const int64_t arch_id = gguf_find_key(gguf_ctx.get(), "general.architecture");
+ if (arch_id < 0 || gguf_get_kv_type(gguf_ctx.get(), arch_id) != GGUF_TYPE_STRING) {
+ return {};
+ }
+
+ const std::string arch = gguf_get_val_str(gguf_ctx.get(), arch_id);
+ if (arch != "dflash") {
+ return {};
+ }
+
+ // the Markov head distinguishes draft-dspark from draft-dflash
+ const auto type = gguf_find_tensor(gguf_ctx.get(), "markov_w1.weight") >= 0
+ ? COMMON_SPECULATIVE_TYPE_DRAFT_DSPARK
+ : COMMON_SPECULATIVE_TYPE_DRAFT_DFLASH;
+
+ SPC_INF("auto-detected speculative type '%s' from the draft model metadata\n", common_speculative_type_to_str(type).c_str());
+
+ return { type };
+}
+
static uint32_t common_get_enabled_speculative_configs(const std::vector<common_speculative_type> & configs) {
uint32_t result = 0;
for (size_t i = 0; i < configs.size(); i++) {
diff --git a/common/speculative.h b/common/speculative.h
index 06b0992ed6..12ae31b7de 100644
--- a/common/speculative.h
+++ b/common/speculative.h
@@ -14,6 +14,9 @@ const char * common_speculative_all_types_str();
// parse user provided types
std::vector<enum common_speculative_type> common_speculative_types_from_names(const std::vector<std::string> & names);
+// infer the spec types from the GGUF metadata of a draft model; empty if unknown
+std::vector<enum common_speculative_type> common_speculative_types_from_gguf(const std::string & path);
+
// convert string to type
enum common_speculative_type common_speculative_type_from_name(const std::string & name);