LLM inference in C/C++
C++ ggml
model: support speculators-format checkpoints for DSpark (#26275)
王金旭9cd719af2107
diff --git a/common/speculative.cpp b/common/speculative.cppindex cd2dfc760b..ae55e357d5 100644--- a/common/speculative.cpp+++ b/common/speculative.cpp@@ -926,6 +926,9 @@ struct common_speculative_impl_draft_dflash : public common_speculative_impl {// draft-dspark: the draft carries a Markov head and uses an anchor-first block layoutconst bool is_dspark;+ // dspark speculators+ bool sample_from_anchor = true;+const int32_t * target_layer_ids = nullptr; // model_dft's extract layer indicesuint32_t target_layer_ids_n = 0;@@ -960,16 +963,20 @@ struct common_speculative_impl_draft_dflash : public common_speculative_impl {if (llama_model_meta_val_str(model_dft, "dflash.block_size", buf, sizeof(buf)) >= 0) {block_size = std::atoi(buf);}+ if (llama_model_meta_val_str(model_dft, "dflash.sample_from_anchor", buf, sizeof(buf)) >= 0) {+ sample_from_anchor = std::strcmp(buf, "true") == 0;+ }}mask_token_id = llama_vocab_mask(llama_model_get_vocab(model_dft));LOG_INF("%s: adding speculative implementation '%s'\n", __func__, common_speculative_type_to_str(type).c_str());LOG_INF("%s: - n_max=%d, n_min=%d, p_min=%.2f\n", __func__, this->params.n_max, this->params.n_min, this->params.p_min);- LOG_INF("%s: - block_size=%d, mask_token_id=%d, n_extract=%u\n", __func__, block_size, mask_token_id, target_layer_ids_n);+ LOG_INF("%s: - block_size=%d, mask_token_id=%d, n_extract=%u, sample_from_anchor=%s\n", __func__,+ block_size, mask_token_id, target_layer_ids_n, sample_from_anchor ? "true" : "false");// DFlash input is [id_last, <mask> * (block_size-1)]: in-place denoising yields at most- // block_size-1 draft tokens, DSpark yield a full block_size draft tokens- const int32_t n_draft_max = is_dspark ? block_size : block_size - 1;+ // block_size-1 draft tokens, anchor-first DSpark yields a full block_size draft tokens+ const int32_t n_draft_max = is_dspark && sample_from_anchor ? block_size : block_size - 1;if (this->params.n_max > n_draft_max || this->params.n_min > n_draft_max) {LOG_WRN("%s: requested draft size (n_max=%d, n_min=%d) exceeds the trained block size %d -- clamping to %d\n",__func__, this->params.n_max, this->params.n_min, block_size, n_draft_max);@@ -1175,7 +1182,7 @@ struct common_speculative_impl_draft_dflash : public common_speculative_impl {const int32_t n_draft = params.n_max;- const int32_t n_block_tokens = n_draft + (is_dspark ? 0 : 1);+ const int32_t n_block_tokens = n_draft + (is_dspark && sample_from_anchor ? 0 : 1);i_block_beg[seq_id] = batch.n_tokens;n_block [seq_id] = n_block_tokens;for (int32_t i = 0; i < n_block_tokens; ++i) {@@ -1208,11 +1215,11 @@ struct common_speculative_impl_draft_dflash : public common_speculative_impl {auto & result = *dp.result;if (is_dspark) {- // DSpark predicts the next token from position 0 and optionally truncates- // at the first position below the confidence threshold.+ // DSpark: read from the first draft slot, truncate below the confidence thresholdconst float * conf = params.p_min > 0.0f ? llama_get_embeddings_nextn(ctx_dft) : nullptr;-- for (int32_t i = 0; i < n_block_tokens; ++i) {+ // bonus-anchor drafts read the mask positions only, like DFlash+ const int32_t i_draft_beg = sample_from_anchor ? 0 : 1;+ for (int32_t i = i_draft_beg; i < n_block_tokens; ++i) {const int32_t idx = beg + i;if (conf && conf[(size_t) idx * n_embd_dec] < params.p_min) {diff --git a/conversion/__init__.py b/conversion/__init__.pyindex 4731b62953..3232a1050b 100644--- a/conversion/__init__.py+++ b/conversion/__init__.py@@ -55,6 +55,8 @@ TEXT_MODEL_MAP: dict[str, str] = {"DeepseekV32ForCausalLM": "deepseek","DFlashDraftModel": "qwen","Qwen3DSparkModel": "qwen",+ "DSparkDraftModel": "qwen",+ "DSparkSpeculator": "qwen","DeepseekV4ForCausalLM": "deepseek","DeepseekV4DSparkModel": "deepseek","DistilBertForMaskedLM": "bert",diff --git a/conversion/qwen.py b/conversion/qwen.pyindex 7a1c11317b..26b10452b6 100644--- a/conversion/qwen.py+++ b/conversion/qwen.py@@ -4,12 +4,13 @@ import jsonfrom typing import Any, Callable, Iterable, TYPE_CHECKING+import numpy as npimport torchif TYPE_CHECKING:from torch import Tensor-from .base import ModelBase, TextModel, gguf, logger+from .base import LazyTorchTensor, ModelBase, TextModel, gguf, logger@ModelBase.register("QWenLMHeadModel")@@ -708,22 +709,82 @@ class DFlashModel(Qwen3Model):yield from super().modify_tensors(data_torch, name, bid)[email protected]("Qwen3DSparkModel")[email protected]("Qwen3DSparkModel", "DSparkDraftModel", "DSparkSpeculator")@ModelBase.example("satgeze/Qwen3.6-27B-DSpark")class DSparkModel(DFlashModel):- # DSpark = DFlash + a semi-autoregressive Markov head+ # DSpark = DFlash + a semi-autoregressive Markov head.model_arch = gguf.MODEL_ARCH.DFLASH- def __init__(self, *args, **kwargs):- super().__init__(*args, **kwargs)- # normalize the flat DeepSpec schema to DFlash's nested dflash_config- self.hparams.setdefault("dflash_config", {- k: self.hparams[k] for k in ("target_layer_ids", "mask_token_id") if k in self.hparams- })+ def __init__(self, dir_model, *args, **kwargs):+ hparams = kwargs.pop("hparams", None)+ if hparams is None:+ hparams = ModelBase.load_hparams(dir_model, False)++ # EAGLE3-style exports use the 1+N bonus-anchor block, DFlash-lineage exports sample from the anchor+ self._sample_from_anchor = hparams.get(+ "sample_from_anchor",+ "transformer_layer_config" not in hparams and "aux_hidden_state_layer_ids" not in hparams)+ if "transformer_layer_config" in hparams:+ hparams = {**hparams, **hparams["transformer_layer_config"]}++ super().__init__(dir_model, *args, hparams=hparams, **kwargs)++ # normalize both schemas to DFlash's nested dflash_config+ if "aux_hidden_state_layer_ids" in self.hparams:+ self.hparams.setdefault("dflash_config", {+ "mask_token_id": self.hparams.get("mask_token_id"),+ "target_layer_ids": [i - 1 for i in self.hparams["aux_hidden_state_layer_ids"]],+ })+ else:+ self.hparams.setdefault("dflash_config", {+ k: self.hparams[k] for k in ("target_layer_ids", "mask_token_id") if k in self.hparams+ })++ if (markov_head_type := self.hparams.get("markov_head_type", "vanilla")) != "vanilla":+ raise ValueError(f"unsupported markov_head_type {markov_head_type!r} (only 'vanilla' is supported)")++ n_vocab = self.hparams["vocab_size"]+ self._n_vocab_draft = self.hparams.get("draft_vocab_size") or n_vocab+ if self._n_vocab_draft > n_vocab:+ raise ValueError(f"draft_vocab_size {self._n_vocab_draft} exceeds vocab_size {n_vocab}")+ self._d2t: Tensor | None = None++ def set_gguf_parameters(self):+ super().set_gguf_parameters()+ self.gguf_writer.add_sample_from_anchor(self._sample_from_anchor)@classmethoddef filter_tensors(cls, item: tuple[str, Callable[[], Tensor]]) -> tuple[str, Callable[[], Tensor]] | None:- name, gen = item- if name.endswith(("embed_tokens.weight", "lm_head.weight")):+ if item[0] == "t2d": # not used at runtimereturn None- return super().filter_tensors((name, gen))+ return super().filter_tensors(item)++ def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]:+ if name == "model.d2t":+ self._d2t = data_torch+ return++ if self._n_vocab_draft == self.hparams["vocab_size"] and name.endswith(("embed_tokens.weight", "lm_head.weight")):+ return++ yield from super().modify_tensors(data_torch, name, bid)++ def prepare_tensors(self):+ super().prepare_tensors()++ n_vocab = self.hparams["vocab_size"]+ if self._n_vocab_draft < n_vocab and self._d2t is None:+ raise ValueError(f"draft_vocab_size {self._n_vocab_draft} < vocab_size {n_vocab} but no d2t table found")++ # write d2t as absolute target token ids+ if self._d2t is not None:+ data = LazyTorchTensor.to_eager(self._d2t).to(torch.int64).cpu().numpy().reshape(-1)+ if data.size != self._n_vocab_draft:+ raise ValueError(f"d2t size {data.size} does not match draft_vocab_size {self._n_vocab_draft}")+ data = data + np.arange(data.size, dtype=np.int64)+ if np.any((data < 0) | (data >= n_vocab)):+ raise ValueError(f"d2t target ids out of range for target vocab size {n_vocab}")+ if np.unique(data).size != data.size:+ raise ValueError("d2t contains duplicate target ids")+ logger.info(f"{'d2t,':<30} --> I64, shape = {{{data.size}}}")+ self.gguf_writer.add_tensor("d2t", data, raw_dtype=gguf.GGMLQuantizationType.I64)diff --git a/docs/speculative.md b/docs/speculative.mdindex 25abef1b60..0f9f8a3d97 100644--- a/docs/speculative.md+++ b/docs/speculative.md@@ -106,6 +106,10 @@ acceptance (from the draft's confidence head, if present) falls below `P` (defauCurrently only drafts with a Qwen3 backbone are supported; support for other backbones(e.g. Gemma4) is planned.+DSpark drafts exported in the [speculators](https://github.com/vllm-project/speculators) format+(for example [`RedHatAI/gemma-4-31B-it-speculator.dspark`](https://huggingface.co/RedHatAI/gemma-4-31B-it-speculator.dspark))+convert the same way.+See:- #25173diff --git a/gguf-py/gguf/constants.py b/gguf-py/gguf/constants.pyindex 74a1a2bd35..d043c9b6ec 100644--- a/gguf-py/gguf/constants.py+++ b/gguf-py/gguf/constants.py@@ -162,6 +162,7 @@ class Keys:TARGET_LAYERS = "{arch}.target_layers"TARGET_HIDDEN_SIZE = "{arch}.target_hidden_size"BLOCK_SIZE = "{arch}.block_size"+ SAMPLE_FROM_ANCHOR = "{arch}.sample_from_anchor"NORM_BEFORE_RESIDUAL = "{arch}.norm_before_residual"NORM_BEFORE_FC = "{arch}.norm_before_fc"@@ -4819,6 +4820,7 @@ MODEL_TENSORS: dict[MODEL_ARCH, list[MODEL_TENSOR]] = {],MODEL_ARCH.DFLASH: [MODEL_TENSOR.TOKEN_EMBD,+ MODEL_TENSOR.OUTPUT,MODEL_TENSOR.OUTPUT_NORM,MODEL_TENSOR.ATTN_NORM,MODEL_TENSOR.ATTN_Q,@@ -4858,6 +4860,7 @@ MODEL_TENSORS: dict[MODEL_ARCH, list[MODEL_TENSOR]] = {MODEL_TENSOR.FFN_UP_SHEXP,MODEL_TENSOR.FC,MODEL_TENSOR.ENC_OUTPUT_NORM,+ MODEL_TENSOR.D2T,# optional DSpark headsMODEL_TENSOR.DSPARK_MARKOV_W1,MODEL_TENSOR.DSPARK_MARKOV_W2,diff --git a/gguf-py/gguf/gguf_writer.py b/gguf-py/gguf/gguf_writer.pyindex 475008411c..9e0914fd86 100644--- a/gguf-py/gguf/gguf_writer.py+++ b/gguf-py/gguf/gguf_writer.py@@ -981,6 +981,9 @@ class GGUFWriter:def add_block_size(self, value: int) -> None:self.add_uint32(Keys.LLM.BLOCK_SIZE.format(arch=self.arch), value)+ def add_sample_from_anchor(self, value: bool) -> None:+ self.add_bool(Keys.LLM.SAMPLE_FROM_ANCHOR.format(arch=self.arch), value)+def add_target_layers(self, value: Sequence[int]) -> None:self.add_array(Keys.LLM.TARGET_LAYERS.format(arch=self.arch), value)diff --git a/gguf-py/gguf/tensor_mapping.py b/gguf-py/gguf/tensor_mapping.pyindex b23bc69153..3292942b41 100644--- a/gguf-py/gguf/tensor_mapping.py+++ b/gguf-py/gguf/tensor_mapping.py@@ -76,14 +76,14 @@ class TensorNameMap:# OutputMODEL_TENSOR.OUTPUT: ("embed_out", # gptneox- "lm_head", # gpt2 mpt falcon llama-hf baichuan qwen mamba dbrx jais nemotron exaone olmoe olmo2 phimoe plamo2+ "lm_head", # gpt2 mpt falcon llama-hf baichuan qwen mamba dbrx jais nemotron exaone olmoe olmo2 phimoe plamo2 llama4"output", # llama-pth bloom internlm2"word_embeddings_for_head", # persimmon"lm_head.linear", # phi2"output_layer", # chatglm"head", # rwkv"head.out", # wavtokenizer- "lm_head", # llama4+ "model.lm_head", # dflash"model.transformer.ff_out", # llada"head.decoder", # modern-bert),diff --git a/src/models/dflash.cpp b/src/models/dflash.cppindex 39e4f5dd8c..5b70a51794 100644--- a/src/models/dflash.cpp+++ b/src/models/dflash.cpp@@ -85,6 +85,16 @@ void llama_model_dflash::load_arch_tensors(llama_model_loader &) {const int64_t n_embd_inp = hparams.n_embd_inp_enc();tok_embd = create_tensor(tn(LLM_TENSOR_TOKEN_EMBD, "weight"), { n_embd, n_vocab }, TENSOR_NOT_REQUIRED);++ // reduced draft vocab (optional): d2t maps draft rows to target token ids+ int64_t n_vocab_draft = n_vocab;+ const struct ggml_tensor * d2t_meta = ml->get_tensor_meta("d2t");+ if (d2t_meta) {+ n_vocab_draft = d2t_meta->ne[0];+ d2t = create_tensor(tn(LLM_TENSOR_D2T), { n_vocab_draft }, 0);+ LLAMA_LOG_INFO("%s: DFlash using d2t mapping (draft_vocab_size = %lld)\n", __func__, (long long) n_vocab_draft);+ }+// DSpark = DFlash + a semi-autoregressive Markov head and Confidence head//// TODO: only Qwen3-style backbones are supported for now; other backbones (e.g. Gemma4)@@ -94,7 +104,7 @@ void llama_model_dflash::load_arch_tensors(llama_model_loader &) {const int64_t dspark_markov_rank = markov_meta->ne[0];dspark_markov_w1 = create_tensor(tn(LLM_TENSOR_DSPARK_MARKOV_W1, "weight"), { dspark_markov_rank, n_vocab }, 0);- dspark_markov_w2 = create_tensor(tn(LLM_TENSOR_DSPARK_MARKOV_W2, "weight"), { dspark_markov_rank, n_vocab }, 0);+ dspark_markov_w2 = create_tensor(tn(LLM_TENSOR_DSPARK_MARKOV_W2, "weight"), { dspark_markov_rank, n_vocab_draft }, 0);dspark_conf_proj = create_tensor(tn(LLM_TENSOR_DSPARK_CONF_PROJ, "weight"), { n_embd + dspark_markov_rank, 1 }, 0);dspark_conf_proj_b = create_tensor(tn(LLM_TENSOR_DSPARK_CONF_PROJ, "bias"), { 1 }, TENSOR_NOT_REQUIRED);@@ -157,6 +167,9 @@ void llama_model_dflash::load_arch_tensors(llama_model_loader &) {return;}+ // optional: reduced-vocab drafts ship their own, full-vocab drafts share the target's via ctx_other+ output = create_tensor(tn(LLM_TENSOR_OUTPUT, "weight"), { n_embd, n_vocab_draft }, TENSOR_NOT_REQUIRED);+for (int i = 0; i < n_layer; ++i) {auto & layer = layers[i];@@ -242,6 +255,11 @@ static void build_dspark_markov_head(llm_graph_context & g, const llama_model &const int64_t block_size = std::stoi(it->second);GGML_ASSERT(block_size > 0);+ // bonus anchor (SpecForge exports): slot 0 is a bonus token, not a prediction slot+ const auto it_anchor = model.gguf_kv.find("dflash.sample_from_anchor");+ const bool sample_from_anchor = it_anchor == model.gguf_kv.end() || it_anchor->second == "true";+ const int64_t i_draft_beg = sample_from_anchor ? 0 : 1;+const int64_t n_blocks = g.ubatch.n_seqs_unq;GGML_ASSERT(n_blocks > 0 && n_tok % n_blocks == 0 && "DSpark markov head requires equal-size blocks");// runtime tokens per block in this ubatch (anchor + drafted positions), bounded by training block_size@@ -263,11 +281,26 @@ static void build_dspark_markov_head(llm_graph_context & g, const llama_model &ggml_tensor * cat = nullptr;ggml_tensor * cat_conf = nullptr;+ if (!sample_from_anchor) {+ // bonus anchor slot: pass the logits through unbiased, pad the (unread) confidence column+ cat = ggml_cont(ctx0, ggml_view_2d(ctx0, base, n_vocab, n_blocks, base_stride, 0));+ cat_conf = ggml_sigmoid(ctx0, ggml_cont(ctx0, ggml_view_2d(ctx0, base, 1, n_blocks, base_stride, 0)));+ }+// TODO: the in-graph chain is greedy (argmax); sampling params affect only the final// token pick, not the Markov conditioning path- for (int64_t i = 0; i < block_drafts; ++i) {+ for (int64_t i = i_draft_beg; i < block_drafts; ++i) {ggml_tensor * w1_prev = ggml_get_rows(ctx0, w1, prev); // [R, n_blocks]- ggml_tensor * bias = ggml_mul_mat(ctx0, w2, w1_prev); // [n_vocab, n_blocks]+ ggml_tensor * bias = ggml_mul_mat(ctx0, w2, w1_prev); // [n_vocab_draft, n_blocks]+ if (model.d2t) {+ // reduced draft vocab: scatter the bias to the target rows (base is -inf on the others)+ const int64_t n_draft_vocab = bias->ne[0];+ ggml_tensor * full = ggml_fill(ctx0, ggml_new_tensor_3d(ctx0, GGML_TYPE_F32, 1, n_vocab, n_blocks), 0.0f);+ bias = ggml_set_rows(ctx0, full,+ ggml_reshape_3d(ctx0, bias, 1, n_draft_vocab, n_blocks),+ ggml_reshape_3d(ctx0, model.d2t, n_draft_vocab, 1, 1));+ bias = ggml_reshape_2d(ctx0, bias, n_vocab, n_blocks);+ }// position i of every block: strided view [n_vocab, n_blocks]ggml_tensor * base_i = ggml_view_2d(ctx0, base, n_vocab, n_blocks, base_stride, i*base->nb[1]);@@ -497,6 +530,22 @@ llama_model_dflash::graph<false>::graph(const llama_model & model, const llm_gra}cur = build_lora_mm(output, cur, output_s);++ // reduced-draft-vocab exports: scatter the draft logits to the target vocabulary via d2t+ if (model.d2t) {+ const int64_t n_draft_vocab = cur->ne[0];+ const int64_t n_outputs = cur->ne[1];+ const int64_t n_vocab = (int64_t) model.vocab.n_tokens();++ GGML_ASSERT(model.d2t->type == GGML_TYPE_I64);+ GGML_ASSERT(model.d2t->ne[0] == n_draft_vocab);++ ggml_tensor * logits = ggml_fill(ctx0, ggml_new_tensor_3d(ctx0, GGML_TYPE_F32, 1, n_vocab, n_outputs), -INFINITY);+ cur = ggml_set_rows(ctx0, logits,+ ggml_reshape_3d(ctx0, cur, 1, n_draft_vocab, n_outputs),+ ggml_reshape_3d(ctx0, model.d2t, n_draft_vocab, 1, 1));+ cur = ggml_reshape_2d(ctx0, cur, n_vocab, n_outputs);+ }cb(cur, "result_output", -1);res->t_logits = cur;