Skip to content

LLM inference in C/C++

C++ ggml

spec : auto-detect mtp draft model type (#27005)

Sigbjørn Skjæret1d2869c6e54d
diff --git a/common/speculative.cpp b/common/speculative.cpp
index 862c4f045c..cd2dfc760b 100644
--- a/common/speculative.cpp
+++ b/common/speculative.cpp
@@ -2246,6 +2246,12 @@ std::vector<common_speculative_type> common_speculative_types_from_gguf(const st
const std::string arch = gguf_get_val_str(gguf_ctx.get(), arch_id);
if (arch != "dflash") {
+ const uint32_t block_count = gguf_get_val_u32(gguf_ctx.get(), gguf_find_key(gguf_ctx.get(), (arch + ".block_count").c_str()));
+
+ if (gguf_find_tensor(gguf_ctx.get(), ("blk." + std::to_string(block_count - 1) + ".nextn.eh_proj.weight").c_str()) >= 0) {
+ return { COMMON_SPECULATIVE_TYPE_DRAFT_MTP };
+ }
+
return {};
}