LLM inference in C/C++
C++ ggml
spec: add DSpark speculative decoding (#25173)
王金旭84075273c82f
diff --git a/common/common.h b/common/common.hindex 2792521836..78b1f416e0 100644--- a/common/common.h+++ b/common/common.h@@ -173,6 +173,7 @@ enum common_speculative_type {COMMON_SPECULATIVE_TYPE_DRAFT_EAGLE3, // Eagle3 speculative decodingCOMMON_SPECULATIVE_TYPE_DRAFT_MTP, // Multi-token predictionCOMMON_SPECULATIVE_TYPE_DRAFT_DFLASH, // DFlash speculative decoding+ COMMON_SPECULATIVE_TYPE_DRAFT_DSPARK, // DSpark speculative decoding (DFlash + Markov head)COMMON_SPECULATIVE_TYPE_NGRAM_SIMPLE, // simple self-speculative decoding based on n-gramsCOMMON_SPECULATIVE_TYPE_NGRAM_MAP_K, // self-speculative decoding with n-gram keys onlyCOMMON_SPECULATIVE_TYPE_NGRAM_MAP_K4V, // self-speculative decoding with n-gram keys and 4 m-gram values@@ -388,7 +389,7 @@ struct common_params_speculative {uint32_t need_n_rs_seq() const {bool needs_rs_seq = std::any_of(types.begin(), types.end(), [&](auto t) {- return t == COMMON_SPECULATIVE_TYPE_DRAFT_MTP || t == COMMON_SPECULATIVE_TYPE_DRAFT_EAGLE3 || t == COMMON_SPECULATIVE_TYPE_DRAFT_DFLASH;+ return t == COMMON_SPECULATIVE_TYPE_DRAFT_MTP || t == COMMON_SPECULATIVE_TYPE_DRAFT_EAGLE3 || t == COMMON_SPECULATIVE_TYPE_DRAFT_DFLASH || t == COMMON_SPECULATIVE_TYPE_DRAFT_DSPARK;});return needs_rs_seq ? draft.n_max : 0u;diff --git a/common/speculative.cpp b/common/speculative.cppindex 3a6c073683..5653a90b88 100644--- a/common/speculative.cpp+++ b/common/speculative.cpp@@ -34,6 +34,7 @@ const std::map<std::string, common_speculative_type> common_speculative_type_fro{"draft-eagle3", COMMON_SPECULATIVE_TYPE_DRAFT_EAGLE3},{"draft-mtp", COMMON_SPECULATIVE_TYPE_DRAFT_MTP},{"draft-dflash", COMMON_SPECULATIVE_TYPE_DRAFT_DFLASH},+ {"draft-dspark", COMMON_SPECULATIVE_TYPE_DRAFT_DSPARK},{"ngram-simple", COMMON_SPECULATIVE_TYPE_NGRAM_SIMPLE},{"ngram-map-k", COMMON_SPECULATIVE_TYPE_NGRAM_MAP_K},{"ngram-map-k4v", COMMON_SPECULATIVE_TYPE_NGRAM_MAP_K4V},@@ -928,15 +929,20 @@ struct common_speculative_impl_draft_dflash : public common_speculative_impl {int32_t block_size = 0;llama_token mask_token_id = 0;+ // draft-dspark: the draft carries a Markov head and uses an anchor-first block layout+ const bool is_dspark;+const int32_t * target_layer_ids = nullptr; // model_dft's extract layer indicesuint32_t target_layer_ids_n = 0;// scratch buffer for concatenated target features [n_tokens, n_embd_enc]std::vector<float> features_buf;- common_speculative_impl_draft_dflash(const common_params_speculative & params, uint32_t n_seq)- : common_speculative_impl(COMMON_SPECULATIVE_TYPE_DRAFT_DFLASH, n_seq)+ common_speculative_impl_draft_dflash(const common_params_speculative & params, uint32_t n_seq,+ common_speculative_type type = COMMON_SPECULATIVE_TYPE_DRAFT_DFLASH)+ : common_speculative_impl(type, n_seq), params(params.draft)+ , is_dspark(type == COMMON_SPECULATIVE_TYPE_DRAFT_DSPARK){auto * ctx_tgt = this->params.ctx_tgt;auto * ctx_dft = this->params.ctx_dft;@@ -963,16 +969,18 @@ struct common_speculative_impl_draft_dflash : public common_speculative_impl {}mask_token_id = llama_vocab_mask(llama_model_get_vocab(model_dft));- LOG_INF("%s: adding speculative implementation 'draft-dflash'\n", __func__);+ LOG_INF("%s: adding speculative implementation '%s'\n", __func__, common_speculative_type_to_str(type).c_str());LOG_INF("%s: - n_max=%d, n_min=%d, p_min=%.2f\n", __func__, this->params.n_max, this->params.n_min, this->params.p_min);LOG_INF("%s: - block_size=%d, mask_token_id=%d, n_extract=%u\n", __func__, block_size, mask_token_id, target_layer_ids_n);- // DFlash input is [id_last, <mask> * (block_size-1)], so it can draft at most block_size-1 tokens per step- if (this->params.n_max > block_size - 1 || this->params.n_min > block_size - 1) {- LOG_WRN("%s: requested draft size (n_max=%d, n_min=%d) exceeds the trained DFlash block size %d -- clamping to %d\n",- __func__, this->params.n_max, this->params.n_min, block_size, block_size - 1);- this->params.n_max = std::min(this->params.n_max, block_size - 1);- this->params.n_min = std::min(this->params.n_min, block_size - 1);+ // DFlash input is [id_last, <mask> * (block_size-1)]: in-place denoising yields at most+ // block_size-1 draft tokens, DSpark yield a full block_size draft tokens+ const int32_t n_draft_max = is_dspark ? block_size : block_size - 1;+ if (this->params.n_max > n_draft_max || this->params.n_min > n_draft_max) {+ LOG_WRN("%s: requested draft size (n_max=%d, n_min=%d) exceeds the trained block size %d -- clamping to %d\n",+ __func__, this->params.n_max, this->params.n_min, block_size, n_draft_max);+ this->params.n_max = std::min(this->params.n_max, n_draft_max);+ this->params.n_min = std::min(this->params.n_min, n_draft_max);}batch = llama_batch_init(llama_n_batch(ctx_dft), 0, n_seq);@@ -1136,12 +1144,9 @@ struct common_speculative_impl_draft_dflash : public common_speculative_impl {const int32_t n = (int32_t) dp.n_past;- int32_t n_draft = params.n_max;- if (dp.n_max > 0) {- n_draft = std::min(n_draft, dp.n_max);- }+ const int32_t n_draft = params.n_max;- const int32_t n_block_tokens = n_draft + 1; // id_last + n_draft * <mask>+ const int32_t n_block_tokens = n_draft + (is_dspark ? 0 : 1);i_block_beg[seq_id] = batch.n_tokens;n_block [seq_id] = n_block_tokens;for (int32_t i = 0; i < n_block_tokens; ++i) {@@ -1173,27 +1178,57 @@ struct common_speculative_impl_draft_dflash : public common_speculative_impl {auto & result = *dp.result;- // greedily read the predicted block at this sequence's noise positions 1..n_block_tokens-1- for (int32_t i = 1; i < n_block_tokens; ++i) {- common_sampler_sample(smpl, ctx_dft, beg + i, true);+ if (is_dspark) {+ // DSpark predicts the next token from position 0 and optionally truncates+ // at the first position below the confidence threshold.+ const float * conf = params.p_min > 0.0f ? llama_get_embeddings_nextn(ctx_dft) : nullptr;- const auto * cur_p = common_sampler_get_candidates(smpl, true);+ for (int32_t i = 0; i < n_block_tokens; ++i) {+ const int32_t idx = beg + i;- for (int k = 0; k < std::min(3, (int) cur_p->size); ++k) {- LOG_DBG(" - seq_id %d, draft candidate %3d, pos %3d: %6d (%8.3f) '%s'\n",- seq_id, k, i - 1, cur_p->data[k].id, cur_p->data[k].p,- common_token_to_piece(ctx_dft, cur_p->data[k].id).c_str());- }+ if (conf && conf[(size_t) idx * n_embd_dec] < params.p_min) {+ break;+ }- const llama_token id = cur_p->data[0].id;+ common_sampler_sample(smpl, ctx_dft, idx, true);- if (cur_p->data[0].p < params.p_min) {- break;+ const auto * cur_p = common_sampler_get_candidates(smpl, true);++ for (int k = 0; k < std::min(3, (int) cur_p->size); ++k) {+ LOG_DBG(" - seq_id %d, draft candidate %3d, pos %3d: %6d (%8.3f) '%s'\n",+ seq_id, k, i, cur_p->data[k].id, cur_p->data[k].p,+ common_token_to_piece(ctx_dft, cur_p->data[k].id).c_str());+ }++ const llama_token id = cur_p->data[0].id;++ common_sampler_accept(smpl, id, true);++ result.push_back(id);}+ } else {+ // greedily read the predicted block at this sequence's noise positions 1..n_block_tokens-1+ for (int32_t i = 1; i < n_block_tokens; ++i) {+ common_sampler_sample(smpl, ctx_dft, beg + i, true);- common_sampler_accept(smpl, id, true);+ const auto * cur_p = common_sampler_get_candidates(smpl, true);- result.push_back(id);+ for (int k = 0; k < std::min(3, (int) cur_p->size); ++k) {+ LOG_DBG(" - seq_id %d, draft candidate %3d, pos %3d: %6d (%8.3f) '%s'\n",+ seq_id, k, i - 1, cur_p->data[k].id, cur_p->data[k].p,+ common_token_to_piece(ctx_dft, cur_p->data[k].id).c_str());+ }++ const llama_token id = cur_p->data[0].id;++ if (cur_p->data[0].p < params.p_min) {+ break;+ }++ common_sampler_accept(smpl, id, true);++ result.push_back(id);+ }}if (result.size() < (size_t) params.n_min) {@@ -2155,6 +2190,7 @@ std::string common_speculative_type_to_str(common_speculative_type type) {case COMMON_SPECULATIVE_TYPE_DRAFT_EAGLE3: return "draft-eagle3";case COMMON_SPECULATIVE_TYPE_DRAFT_MTP: return "draft-mtp";case COMMON_SPECULATIVE_TYPE_DRAFT_DFLASH: return "draft-dflash";+ case COMMON_SPECULATIVE_TYPE_DRAFT_DSPARK: return "draft-dspark";case COMMON_SPECULATIVE_TYPE_NGRAM_SIMPLE: return "ngram-simple";case COMMON_SPECULATIVE_TYPE_NGRAM_MAP_K: return "ngram-map-k";case COMMON_SPECULATIVE_TYPE_NGRAM_MAP_K4V: return "ngram-map-k4v";@@ -2208,6 +2244,7 @@ int32_t common_speculative_n_max(const common_params_speculative * spec) {case COMMON_SPECULATIVE_TYPE_DRAFT_EAGLE3:case COMMON_SPECULATIVE_TYPE_DRAFT_MTP:case COMMON_SPECULATIVE_TYPE_DRAFT_DFLASH:+ case COMMON_SPECULATIVE_TYPE_DRAFT_DSPARK:n_max = std::max(n_max, std::max(0, spec->draft.n_max));break;case COMMON_SPECULATIVE_TYPE_NGRAM_SIMPLE:@@ -2352,6 +2389,7 @@ common_speculative * common_speculative_init(common_params_speculative & params,bool has_draft_eagle3 = (enabled_configs & (1u << COMMON_SPECULATIVE_TYPE_DRAFT_EAGLE3)) && params.draft.ctx_dft != nullptr;bool has_draft_mtp = (enabled_configs & (1u << COMMON_SPECULATIVE_TYPE_DRAFT_MTP)) && params.draft.ctx_dft != nullptr;bool has_draft_dflash = (enabled_configs & (1u << COMMON_SPECULATIVE_TYPE_DRAFT_DFLASH)) && params.draft.ctx_dft != nullptr;+ bool has_draft_dspark = (enabled_configs & (1u << COMMON_SPECULATIVE_TYPE_DRAFT_DSPARK)) && params.draft.ctx_dft != nullptr;@@ -2362,7 +2400,7 @@ common_speculative * common_speculative_init(common_params_speculative & params,bool has_ngram_mod = (enabled_configs & (1u << COMMON_SPECULATIVE_TYPE_NGRAM_MOD));// when adding a new type - update here the logic above- static_assert(COMMON_SPECULATIVE_TYPE_COUNT == 10);+ static_assert(COMMON_SPECULATIVE_TYPE_COUNT == 11);// this list here defines the priority of the speculators// the one with highest priority are listed first@@ -2395,6 +2433,9 @@ common_speculative * common_speculative_init(common_params_speculative & params,if (has_draft_dflash) {configs.push_back(common_speculative_config(COMMON_SPECULATIVE_TYPE_DRAFT_DFLASH, params));}+ if (has_draft_dspark) {+ configs.push_back(common_speculative_config(COMMON_SPECULATIVE_TYPE_DRAFT_DSPARK, params));+ }}std::vector<std::unique_ptr<common_speculative_impl>> impls = {};@@ -2419,6 +2460,11 @@ common_speculative * common_speculative_init(common_params_speculative & params,impls.push_back(std::make_unique<common_speculative_impl_draft_dflash>(config.params, n_seq));break;}+ case COMMON_SPECULATIVE_TYPE_DRAFT_DSPARK: {+ impls.push_back(std::make_unique<common_speculative_impl_draft_dflash>(+ config.params, n_seq, COMMON_SPECULATIVE_TYPE_DRAFT_DSPARK));+ break;+ }case COMMON_SPECULATIVE_TYPE_NGRAM_SIMPLE: {common_ngram_map ngram_map = get_common_ngram_map(config.type, config.params.ngram_simple);diff --git a/conversion/__init__.py b/conversion/__init__.pyindex 45c001b78f..1a47b851a0 100644--- a/conversion/__init__.py+++ b/conversion/__init__.py@@ -53,6 +53,7 @@ TEXT_MODEL_MAP: dict[str, str] = {"DeepseekV3ForCausalLM": "deepseek","DeepseekV32ForCausalLM": "deepseek","DFlashDraftModel": "qwen",+ "Qwen3DSparkModel": "qwen","DeepseekV4ForCausalLM": "deepseek","DistilBertForMaskedLM": "bert","DistilBertForSequenceClassification": "bert",diff --git a/conversion/qwen.py b/conversion/qwen.pyindex 9bc2b99fde..d1127f7431 100644--- a/conversion/qwen.py+++ b/conversion/qwen.py@@ -688,3 +688,23 @@ class DFlashModel(Qwen3Model):if not name.startswith("model."):name = "model." + namereturn super().filter_tensors((name, gen))++[email protected]("Qwen3DSparkModel")+class DSparkModel(DFlashModel):+ # DSpark = DFlash + a semi-autoregressive Markov head+ model_arch = gguf.MODEL_ARCH.DFLASH++ def __init__(self, *args, **kwargs):+ super().__init__(*args, **kwargs)+ # normalize the flat DeepSpec schema to DFlash's nested dflash_config+ self.hparams.setdefault("dflash_config", {+ k: self.hparams[k] for k in ("target_layer_ids", "mask_token_id") if k in self.hparams+ })++ @classmethod+ def filter_tensors(cls, item: tuple[str, Callable[[], Tensor]]) -> tuple[str, Callable[[], Tensor]] | None:+ name, gen = item+ if name.endswith(("embed_tokens.weight", "lm_head.weight")):+ return None+ return super().filter_tensors((name, gen))diff --git a/docs/speculative.md b/docs/speculative.mdindex 4100b92f8f..3957db85c9 100644--- a/docs/speculative.md+++ b/docs/speculative.md@@ -78,6 +78,38 @@ See:- #22105+### DSpark (`draft-dspark`)++DSpark extends DFlash with a semi-autoregressive _Markov head_: the draft still emits a whole+block per forward pass, but each block position's logits are biased by a low-rank term keyed on+the previous token, chained in-graph across the block. This keeps drafting at one decode per+block while recovering some of the left-to-right signal that pure block diffusion loses.++The draft is a small DeepSpec checkpoint trained for a specific target (for example+[`deepseek-ai/dspark_qwen3_4b_block7`](https://huggingface.co/deepseek-ai/dspark_qwen3_4b_block7)+for `Qwen/Qwen3-4B`). Convert it with `--target-model-dir` so it inherits the target's tokenizer+and token embeddings:++```bash+python convert_hf_to_gguf.py deepseek-ai/dspark_qwen3_4b_block7 \+ --target-model-dir Qwen/Qwen3-4B --outtype bf16 --outfile Qwen3-4B-DSpark.gguf++llama-server -m Qwen3-4B.gguf -md Qwen3-4B-DSpark.gguf \+ --spec-type draft-dspark --spec-draft-n-max 7 -fa on --jinja+```++`--spec-draft-n-max` is clamped to the draft model's trained block size.++`--spec-draft-conf-min P` truncates each drafted block at the first position whose predicted+acceptance (from the draft's confidence head, if present) falls below `P` (default 0 = disabled).++Currently only drafts with a Qwen3 backbone are supported; support for other backbones+(e.g. Gemma4) is planned.++See:++- #25173+### n-gram Cache (`ngram-cache`)An n-gram is a sequence of n tokens. The n-gram cache implementation maintains statistics about short n-gram sequences.@@ -173,7 +205,7 @@ If a draft model is combined with a draftless decoding the draftless decoding ha### General Speculative Parameters```---spec-type [none|draft-simple|draft-eagle3|draft-dflash|draft-mtp|ngram-cache|ngram-simple|ngram-map-k|ngram-map-k4v|ngram-mod]+--spec-type [none|draft-simple|draft-eagle3|draft-dflash|draft-dspark|draft-mtp|ngram-cache|ngram-simple|ngram-map-k|ngram-map-k4v|ngram-mod]comma-separated list of types of speculative decoding to use(default: none)(env: LLAMA_ARG_SPEC_TYPE)@@ -314,6 +346,7 @@ Specifies a comma-separated list of speculative decoding types to use.| `draft-simple` | Use a simple draft model for speculation || `draft-eagle3` | Use an EAGLE-3 draft model that reads the target's hidden states || `draft-dflash` | Use a DFlash block-diffusion draft model that emits a block per step |+| `draft-dspark` | Use a DSpark draft model (DFlash backbone + semi-autoregressive Markov head) || `draft-mtp` | Use Multi Token Prediction (MTP) heads from the main model || `ngram-cache` | Use n-gram cache lookup || `ngram-simple` | Use simple n-gram pattern matching |diff --git a/gguf-py/gguf/constants.py b/gguf-py/gguf/constants.pyindex 4df94b9a3a..2ebefaa3b4 100644--- a/gguf-py/gguf/constants.py+++ b/gguf-py/gguf/constants.py@@ -988,6 +988,10 @@ class MODEL_TENSOR(IntEnum):# eagle3FC = auto() # feature fusion layerD2T = auto() # draft to target vocabulary mapping+ # dspark+ DSPARK_MARKOV_W1 = auto() # markov head: prev-token embed+ DSPARK_MARKOV_W2 = auto() # markov head: bias projection+ DSPARK_CONF_PROJ = auto() # confidence head# lfm2 audioA_ENC_NORM_CONV = auto()A_ENC_LINEAR_POS = auto()@@ -1617,6 +1621,9 @@ TENSOR_NAMES: dict[MODEL_TENSOR, str] = {MODEL_TENSOR.NEXTN_SHARED_HEAD_HEAD: "blk.{bid}.nextn.shared_head_head",MODEL_TENSOR.NEXTN_SHARED_HEAD_NORM: "blk.{bid}.nextn.shared_head_norm",MODEL_TENSOR.FC: "fc",+ MODEL_TENSOR.DSPARK_MARKOV_W1: "markov_w1",+ MODEL_TENSOR.DSPARK_MARKOV_W2: "markov_w2",+ MODEL_TENSOR.DSPARK_CONF_PROJ: "conf_proj",MODEL_TENSOR.D2T: "d2t",}@@ -4362,6 +4369,10 @@ MODEL_TENSORS: dict[MODEL_ARCH, list[MODEL_TENSOR]] = {MODEL_TENSOR.FFN_UP,MODEL_TENSOR.FC,MODEL_TENSOR.ENC_OUTPUT_NORM,+ # optional DSpark heads+ MODEL_TENSOR.DSPARK_MARKOV_W1,+ MODEL_TENSOR.DSPARK_MARKOV_W2,+ MODEL_TENSOR.DSPARK_CONF_PROJ,],MODEL_ARCH.MISTRAL4: [MODEL_TENSOR.TOKEN_EMBD,diff --git a/gguf-py/gguf/tensor_mapping.py b/gguf-py/gguf/tensor_mapping.pyindex 8299ac25b4..5562d43277 100644--- a/gguf-py/gguf/tensor_mapping.py+++ b/gguf-py/gguf/tensor_mapping.py@@ -1304,6 +1304,18 @@ class TensorNameMap:"model.fc", # dflash),+ MODEL_TENSOR.DSPARK_MARKOV_W1: (+ "model.markov_head.markov_w1", # dspark+ ),++ MODEL_TENSOR.DSPARK_MARKOV_W2: (+ "model.markov_head.markov_w2", # dspark+ ),++ MODEL_TENSOR.DSPARK_CONF_PROJ: (+ "model.confidence_head.proj", # dspark+ ),+MODEL_TENSOR.CLS: ("classifier", # jina"classifier.dense", # robertadiff --git a/src/llama-arch.cpp b/src/llama-arch.cppindex a0945e501d..e81ff647ee 100644--- a/src/llama-arch.cpp+++ b/src/llama-arch.cpp@@ -616,6 +616,9 @@ static const std::map<llm_tensor, const char *> LLM_TENSOR_NAMES = {{ LLM_TENSOR_MASKED_EMBD_ORDERING, "masked_embd_ordering" },{ LLM_TENSOR_FC, "fc" },{ LLM_TENSOR_D2T, "d2t" },+ { LLM_TENSOR_DSPARK_MARKOV_W1, "markov_w1" },+ { LLM_TENSOR_DSPARK_MARKOV_W2, "markov_w2" },+ { LLM_TENSOR_DSPARK_CONF_PROJ, "conf_proj" },};// declare information about the model weight tensors:@@ -870,6 +873,10 @@ static const std::map<llm_tensor, llm_tensor_info> LLM_TENSOR_INFOS = {// eagle3{LLM_TENSOR_FC, {LLM_TENSOR_LAYER_OUTPUT, GGML_OP_MUL_MAT}},{LLM_TENSOR_D2T, {LLM_TENSOR_LAYER_OUTPUT, GGML_OP_GET_ROWS}},+ // dspark+ {LLM_TENSOR_DSPARK_MARKOV_W1, {LLM_TENSOR_LAYER_OUTPUT, GGML_OP_GET_ROWS}},+ {LLM_TENSOR_DSPARK_MARKOV_W2, {LLM_TENSOR_LAYER_OUTPUT, GGML_OP_MUL_MAT}},+ {LLM_TENSOR_DSPARK_CONF_PROJ, {LLM_TENSOR_LAYER_OUTPUT, GGML_OP_MUL_MAT}},};LLM_KV::LLM_KV(llm_arch arch, const char * suffix) : arch(arch), suffix(suffix) {}diff --git a/src/llama-arch.h b/src/llama-arch.hindex fb5eb3920b..cbc97085ea 100644--- a/src/llama-arch.h+++ b/src/llama-arch.h@@ -624,6 +624,9 @@ enum llm_tensor {LLM_TENSOR_MASKED_EMBD_ORDERING,LLM_TENSOR_FC,LLM_TENSOR_D2T,+ LLM_TENSOR_DSPARK_MARKOV_W1,+ LLM_TENSOR_DSPARK_MARKOV_W2,+ LLM_TENSOR_DSPARK_CONF_PROJ,};diff --git a/src/llama-model.h b/src/llama-model.hindex 36d0480e5e..d6a40fa302 100644--- a/src/llama-model.h+++ b/src/llama-model.h@@ -606,6 +606,12 @@ struct llama_model {struct ggml_tensor * fc = nullptr; // feature fusion layerstruct ggml_tensor * d2t = nullptr; // draft to target vocabulary mapping+ // dspark+ struct ggml_tensor * dspark_markov_w1 = nullptr;+ struct ggml_tensor * dspark_markov_w2 = nullptr;+ struct ggml_tensor * dspark_conf_proj = nullptr;+ struct ggml_tensor * dspark_conf_proj_b = nullptr;+// unified vector to store target-model extracted layer ids in eagle3, dflash, etc.std::vector<int32_t> target_layer_ids;diff --git a/src/models/dflash.cpp b/src/models/dflash.cppindex 427eed4594..dcff3aec9f 100644--- a/src/models/dflash.cpp+++ b/src/models/dflash.cpp@@ -37,6 +37,23 @@ void llama_model_dflash::load_arch_tensors(llama_model_loader &) {const int64_t n_embd_inp = hparams.n_embd_inp_enc();+ // DSpark = DFlash + a semi-autoregressive Markov head and Confidence head+ //+ // TODO: only Qwen3-style backbones are supported for now; other backbones (e.g. Gemma4)+ // need their own conversion path and graph tweaks+ const struct ggml_tensor * markov_meta = ml->get_tensor_meta("markov_w1.weight");+ if (markov_meta) {+ const int64_t dspark_markov_rank = markov_meta->ne[0];++ dspark_markov_w1 = create_tensor(tn(LLM_TENSOR_DSPARK_MARKOV_W1, "weight"), { dspark_markov_rank, n_vocab }, 0);+ dspark_markov_w2 = create_tensor(tn(LLM_TENSOR_DSPARK_MARKOV_W2, "weight"), { dspark_markov_rank, n_vocab }, 0);++ dspark_conf_proj = create_tensor(tn(LLM_TENSOR_DSPARK_CONF_PROJ, "weight"), { n_embd + dspark_markov_rank, 1 }, 0);+ dspark_conf_proj_b = create_tensor(tn(LLM_TENSOR_DSPARK_CONF_PROJ, "bias"), { 1 }, TENSOR_NOT_REQUIRED);++ LLAMA_LOG_INFO("%s: DFlash with DSpark markov head (rank = %lld)\n", __func__, (long long) dspark_markov_rank);+ }+fc = create_tensor(tn(LLM_TENSOR_FC, "weight"), { n_embd_inp, n_embd }, 0);output_norm_enc = create_tensor(tn(LLM_TENSOR_ENC_OUTPUT_NORM, "weight"), { n_embd }, 0); // encoder hidden_norm (after fc)output_norm = create_tensor(tn(LLM_TENSOR_OUTPUT_NORM, "weight"), { n_embd }, 0); // decoder final norm@@ -105,6 +122,94 @@ llama_model_dflash::graph<true>::graph(const llama_model & model, const llm_grapggml_build_forward_expand(gf, cur);}+// DSpark (DFlash + Markov & Confidence head): Markov bias on the draft logits, chained per block position+static void build_dspark_markov_head(llm_graph_context & g, const llama_model & model, ggml_tensor * tokens) {+ ggml_context * ctx0 = g.ctx0;+ auto & res = g.res;++ ggml_tensor * w1 = model.dspark_markov_w1;+ ggml_tensor * w2 = model.dspark_markov_w2;+ GGML_ASSERT(w1 && w2 && model.dspark_conf_proj && "DSpark markov/confidence weights not loaded");++ ggml_tensor * base = res->t_logits; // [n_vocab, n_tokens]+ const int64_t n_vocab = base->ne[0];+ const int64_t n_tok = base->ne[1];++ const auto it = model.gguf_kv.find("dflash.block_size");+ GGML_ASSERT(it != model.gguf_kv.end() && "DSpark draft requires 'dflash.block_size' in GGUF metadata");+ const int64_t block_size = std::stoi(it->second);+ GGML_ASSERT(block_size > 0);++ const int64_t n_blocks = g.ubatch.n_seqs_unq;+ GGML_ASSERT(n_blocks > 0 && n_tok % n_blocks == 0 && "DSpark markov head requires equal-size blocks");+ // runtime tokens per block in this ubatch (anchor + drafted positions), bounded by training block_size+ const int64_t block_drafts = n_tok / n_blocks;+ if (block_drafts > block_size) {+ return;+ }++ // anchor (committed last) token of every block: token 0 of each block, i.e. a strided view+ const size_t token_stride = (size_t) block_drafts * tokens->nb[0];+ const size_t base_stride = (size_t) block_drafts * base->nb[1];++ ggml_tensor * prev = ggml_view_2d(ctx0, tokens, 1, n_blocks, token_stride, 0);+ prev = ggml_cont_1d(ctx0, prev, n_blocks);++ // confidence head input: predicts per-position acceptance+ ggml_tensor * conf_inp = res->t_embd; // [n_embd, n_tok]++ ggml_tensor * cat = nullptr;+ ggml_tensor * cat_conf = nullptr;++ // TODO: the in-graph chain is greedy (argmax); sampling params affect only the final+ // token pick, not the Markov conditioning path+ for (int64_t i = 0; i < block_drafts; ++i) {+ ggml_tensor * w1_prev = ggml_get_rows(ctx0, w1, prev); // [R, n_blocks]+ ggml_tensor * bias = ggml_mul_mat(ctx0, w2, w1_prev); // [n_vocab, n_blocks]++ // position i of every block: strided view [n_vocab, n_blocks]+ ggml_tensor * base_i = ggml_view_2d(ctx0, base, n_vocab, n_blocks, base_stride, i*base->nb[1]);+ ggml_tensor * col = ggml_add(ctx0, base_i, bias);++ cat = cat ? ggml_concat(ctx0, cat, col, 1) : col;++ // conf(i) = sigmoid(conf_proj . [conf_inp(i); markov_w1[prev(i)]] + b) -- [1, n_blocks]+ ggml_tensor * conf_inp_i = ggml_view_2d(ctx0, conf_inp, conf_inp->ne[0], n_blocks,+ (size_t) block_drafts * conf_inp->nb[1], i*conf_inp->nb[1]);+ ggml_tensor * feat = ggml_concat(ctx0, ggml_cont(ctx0, conf_inp_i), w1_prev, 0);+ ggml_tensor * conf = ggml_mul_mat(ctx0, model.dspark_conf_proj, feat);+ if (model.dspark_conf_proj_b) {+ conf = ggml_add(ctx0, conf, model.dspark_conf_proj_b);+ }+ conf = ggml_sigmoid(ctx0, conf);++ cat_conf = cat_conf ? ggml_concat(ctx0, cat_conf, conf, 1) : conf;++ if (i + 1 < block_drafts) {+ prev = ggml_argmax(ctx0, col);+ }+ }++ // cat is position-major; restore ubatch block-major order+ ggml_tensor * out = ggml_reshape_3d(ctx0, cat, n_vocab, n_blocks, block_drafts);+ out = ggml_cont(ctx0, ggml_permute(ctx0, out, 0, 2, 1, 3)); // [n_vocab, block_drafts, n_blocks]+ out = ggml_reshape_2d(ctx0, out, n_vocab, n_tok);++ {+ ggml_tensor * conf = ggml_reshape_3d(ctx0, cat_conf, 1, n_blocks, block_drafts);+ conf = ggml_cont(ctx0, ggml_permute(ctx0, conf, 0, 2, 1, 3));+ conf = ggml_reshape_2d(ctx0, conf, 1, n_tok);++ // note: broadcast the [1, n_tok] confidences to n_embd-wide rows to be able to reuse `llama_get_embeddings_nextn`+ conf = ggml_repeat(ctx0, conf, res->t_embd);+ res->t_h_nextn = conf;+ ggml_build_forward_expand(g.gf, conf);+ }++ res->t_logits = out;+ ggml_build_forward_expand(g.gf, out);+}+// DFlash decoder, dual-mode by batch type:// * embd batch -> fused target features: project + inject K/V into the cache.// * token batch -> noise-block diffusion: attend over [committed, MASK...] to generate draft tokens@@ -210,6 +315,8 @@ llama_model_dflash::graph<false>::graph(const llama_model & model, const llm_grainp->tokens = ggml_new_tensor_1d(ctx0, GGML_TYPE_I32, n_tokens);ggml_set_input(inp->tokens);+ ggml_tensor * inp_tokens = inp->tokens;+ggml_tensor * inpL = ggml_get_rows(ctx0, tok_embd, inp->tokens);cb(inpL, "inp_noise_embd", -1);@@ -290,4 +397,9 @@ llama_model_dflash::graph<false>::graph(const llama_model & model, const llm_grares->t_logits = cur;ggml_build_forward_expand(gf, cur);++ // DSpark: bias the draft logits with the Markov head+ if (model.dspark_markov_w1) {+ build_dspark_markov_head(*this, model, inp_tokens);+ }}diff --git a/tools/cli/README.md b/tools/cli/README.mdindex 972ea04dc7..bcddd05702 100644--- a/tools/cli/README.md+++ b/tools/cli/README.md@@ -203,7 +203,7 @@| `--spec-draft-device, -devd, --device-draft <dev1,dev2,..>` | comma-separated list of devices to use for offloading the draft model (none = don't offload)<br/>use --list-devices to see a list of available devices || `--spec-draft-ngl, -ngld, --gpu-layers-draft, --n-gpu-layers-draft N` | max. number of draft model layers to store in VRAM, either an exact number, 'auto', or 'all' (default: auto)<br/>(env: LLAMA_ARG_N_GPU_LAYERS_DRAFT) || `--spec-draft-model, -md, --model-draft FNAME` | draft model for speculative decoding (default: unused)<br/>(env: LLAMA_ARG_SPEC_DRAFT_MODEL) |-| `--spec-type none,draft-simple,draft-eagle3,draft-mtp,draft-dflash,ngram-simple,ngram-map-k,ngram-map-k4v,ngram-mod,ngram-cache` | comma-separated list of types of speculative decoding to use (default: none)<br/><br/>(env: LLAMA_ARG_SPEC_TYPE) |+| `--spec-type none,draft-simple,draft-eagle3,draft-mtp,draft-dflash,draft-dspark,ngram-simple,ngram-map-k,ngram-map-k4v,ngram-mod,ngram-cache` | comma-separated list of types of speculative decoding to use (default: none)<br/><br/>(env: LLAMA_ARG_SPEC_TYPE) || `--spec-ngram-mod-n-min N` | minimum number of ngram tokens to use for ngram-based speculative decoding (default: 48) || `--spec-ngram-mod-n-max N` | maximum number of ngram tokens to use for ngram-based speculative decoding (default: 64) || `--spec-ngram-mod-n-match N` | ngram-mod lookup length (default: 24) |diff --git a/tools/server/README.md b/tools/server/README.mdindex 25aacf9f51..f45c018972 100644--- a/tools/server/README.md+++ b/tools/server/README.md@@ -259,7 +259,7 @@ For the full list of features, please refer to [server's changelog](https://gith| `--spec-draft-device, -devd, --device-draft <dev1,dev2,..>` | comma-separated list of devices to use for offloading the draft model (none = don't offload)<br/>use --list-devices to see a list of available devices || `--spec-draft-ngl, -ngld, --gpu-layers-draft, --n-gpu-layers-draft N` | max. number of draft model layers to store in VRAM, either an exact number, 'auto', or 'all' (default: auto)<br/>(env: LLAMA_ARG_N_GPU_LAYERS_DRAFT) || `--spec-draft-model, -md, --model-draft FNAME` | draft model for speculative decoding (default: unused)<br/>(env: LLAMA_ARG_SPEC_DRAFT_MODEL) |-| `--spec-type none,draft-simple,draft-eagle3,draft-mtp,draft-dflash,ngram-simple,ngram-map-k,ngram-map-k4v,ngram-mod,ngram-cache` | comma-separated list of types of speculative decoding to use (default: none)<br/><br/>(env: LLAMA_ARG_SPEC_TYPE) |+| `--spec-type none,draft-simple,draft-eagle3,draft-mtp,draft-dflash,draft-dspark,ngram-simple,ngram-map-k,ngram-map-k4v,ngram-mod,ngram-cache` | comma-separated list of types of speculative decoding to use (default: none)<br/><br/>(env: LLAMA_ARG_SPEC_TYPE) || `--spec-ngram-mod-n-min N` | minimum number of ngram tokens to use for ngram-based speculative decoding (default: 48) || `--spec-ngram-mod-n-max N` | maximum number of ngram tokens to use for ngram-based speculative decoding (default: 64) || `--spec-ngram-mod-n-match N` | ngram-mod lookup length (default: 24) |diff --git a/tools/server/server-schema.cpp b/tools/server/server-schema.cppindex e880f4ca72..674d3ba337 100644--- a/tools/server/server-schema.cpp+++ b/tools/server/server-schema.cpp@@ -209,6 +209,7 @@ std::vector<std::unique_ptr<field>> make_llama_cmpl_schema(const common_params &->set_hard_limits(0.0f, 1.0f)->set_desc("Minimum speculative decoding probability for draft tokens (0 = greedy)"));+add((new field_str("speculative.type"))->set_desc("Speculative decoding method (for debugging and research purposes)")->set_handler([&](field_eval_context & ctx, const json & data) {