Skip to content

LLM inference in C/C++

C++ ggml

spec: add DSpark speculative decoding (#25173)

王金旭84075273c82f
diff --git a/common/common.h b/common/common.h
index 2792521836..78b1f416e0 100644
--- a/common/common.h
+++ b/common/common.h
@@ -173,6 +173,7 @@ enum common_speculative_type {
COMMON_SPECULATIVE_TYPE_DRAFT_EAGLE3, // Eagle3 speculative decoding
COMMON_SPECULATIVE_TYPE_DRAFT_MTP, // Multi-token prediction
COMMON_SPECULATIVE_TYPE_DRAFT_DFLASH, // DFlash speculative decoding
+ COMMON_SPECULATIVE_TYPE_DRAFT_DSPARK, // DSpark speculative decoding (DFlash + Markov head)
COMMON_SPECULATIVE_TYPE_NGRAM_SIMPLE, // simple self-speculative decoding based on n-grams
COMMON_SPECULATIVE_TYPE_NGRAM_MAP_K, // self-speculative decoding with n-gram keys only
COMMON_SPECULATIVE_TYPE_NGRAM_MAP_K4V, // self-speculative decoding with n-gram keys and 4 m-gram values
@@ -388,7 +389,7 @@ struct common_params_speculative {
uint32_t need_n_rs_seq() const {
bool needs_rs_seq = std::any_of(types.begin(), types.end(), [&](auto t) {
- return t == COMMON_SPECULATIVE_TYPE_DRAFT_MTP || t == COMMON_SPECULATIVE_TYPE_DRAFT_EAGLE3 || t == COMMON_SPECULATIVE_TYPE_DRAFT_DFLASH;
+ return t == COMMON_SPECULATIVE_TYPE_DRAFT_MTP || t == COMMON_SPECULATIVE_TYPE_DRAFT_EAGLE3 || t == COMMON_SPECULATIVE_TYPE_DRAFT_DFLASH || t == COMMON_SPECULATIVE_TYPE_DRAFT_DSPARK;
});
return needs_rs_seq ? draft.n_max : 0u;
diff --git a/common/speculative.cpp b/common/speculative.cpp
index 3a6c073683..5653a90b88 100644
--- a/common/speculative.cpp
+++ b/common/speculative.cpp
@@ -34,6 +34,7 @@ const std::map<std::string, common_speculative_type> common_speculative_type_fro
{"draft-eagle3", COMMON_SPECULATIVE_TYPE_DRAFT_EAGLE3},
{"draft-mtp", COMMON_SPECULATIVE_TYPE_DRAFT_MTP},
{"draft-dflash", COMMON_SPECULATIVE_TYPE_DRAFT_DFLASH},
+ {"draft-dspark", COMMON_SPECULATIVE_TYPE_DRAFT_DSPARK},
{"ngram-simple", COMMON_SPECULATIVE_TYPE_NGRAM_SIMPLE},
{"ngram-map-k", COMMON_SPECULATIVE_TYPE_NGRAM_MAP_K},
{"ngram-map-k4v", COMMON_SPECULATIVE_TYPE_NGRAM_MAP_K4V},
@@ -928,15 +929,20 @@ struct common_speculative_impl_draft_dflash : public common_speculative_impl {
int32_t block_size = 0;
llama_token mask_token_id = 0;
+ // draft-dspark: the draft carries a Markov head and uses an anchor-first block layout
+ const bool is_dspark;
+
const int32_t * target_layer_ids = nullptr; // model_dft's extract layer indices
uint32_t target_layer_ids_n = 0;
// scratch buffer for concatenated target features [n_tokens, n_embd_enc]
std::vector<float> features_buf;
- common_speculative_impl_draft_dflash(const common_params_speculative & params, uint32_t n_seq)
- : common_speculative_impl(COMMON_SPECULATIVE_TYPE_DRAFT_DFLASH, n_seq)
+ common_speculative_impl_draft_dflash(const common_params_speculative & params, uint32_t n_seq,
+ common_speculative_type type = COMMON_SPECULATIVE_TYPE_DRAFT_DFLASH)
+ : common_speculative_impl(type, n_seq)
, params(params.draft)
+ , is_dspark(type == COMMON_SPECULATIVE_TYPE_DRAFT_DSPARK)
{
auto * ctx_tgt = this->params.ctx_tgt;
auto * ctx_dft = this->params.ctx_dft;
@@ -963,16 +969,18 @@ struct common_speculative_impl_draft_dflash : public common_speculative_impl {
}
mask_token_id = llama_vocab_mask(llama_model_get_vocab(model_dft));
- LOG_INF("%s: adding speculative implementation 'draft-dflash'\n", __func__);
+ LOG_INF("%s: adding speculative implementation '%s'\n", __func__, common_speculative_type_to_str(type).c_str());
LOG_INF("%s: - n_max=%d, n_min=%d, p_min=%.2f\n", __func__, this->params.n_max, this->params.n_min, this->params.p_min);
LOG_INF("%s: - block_size=%d, mask_token_id=%d, n_extract=%u\n", __func__, block_size, mask_token_id, target_layer_ids_n);
- // DFlash input is [id_last, <mask> * (block_size-1)], so it can draft at most block_size-1 tokens per step
- if (this->params.n_max > block_size - 1 || this->params.n_min > block_size - 1) {
- LOG_WRN("%s: requested draft size (n_max=%d, n_min=%d) exceeds the trained DFlash block size %d -- clamping to %d\n",
- __func__, this->params.n_max, this->params.n_min, block_size, block_size - 1);
- this->params.n_max = std::min(this->params.n_max, block_size - 1);
- this->params.n_min = std::min(this->params.n_min, block_size - 1);
+ // DFlash input is [id_last, <mask> * (block_size-1)]: in-place denoising yields at most
+ // block_size-1 draft tokens, DSpark yield a full block_size draft tokens
+ const int32_t n_draft_max = is_dspark ? block_size : block_size - 1;
+ if (this->params.n_max > n_draft_max || this->params.n_min > n_draft_max) {
+ LOG_WRN("%s: requested draft size (n_max=%d, n_min=%d) exceeds the trained block size %d -- clamping to %d\n",
+ __func__, this->params.n_max, this->params.n_min, block_size, n_draft_max);
+ this->params.n_max = std::min(this->params.n_max, n_draft_max);
+ this->params.n_min = std::min(this->params.n_min, n_draft_max);
}
batch = llama_batch_init(llama_n_batch(ctx_dft), 0, n_seq);
@@ -1136,12 +1144,9 @@ struct common_speculative_impl_draft_dflash : public common_speculative_impl {
const int32_t n = (int32_t) dp.n_past;
- int32_t n_draft = params.n_max;
- if (dp.n_max > 0) {
- n_draft = std::min(n_draft, dp.n_max);
- }
+ const int32_t n_draft = params.n_max;
- const int32_t n_block_tokens = n_draft + 1; // id_last + n_draft * <mask>
+ const int32_t n_block_tokens = n_draft + (is_dspark ? 0 : 1);
i_block_beg[seq_id] = batch.n_tokens;
n_block [seq_id] = n_block_tokens;
for (int32_t i = 0; i < n_block_tokens; ++i) {
@@ -1173,27 +1178,57 @@ struct common_speculative_impl_draft_dflash : public common_speculative_impl {
auto & result = *dp.result;
- // greedily read the predicted block at this sequence's noise positions 1..n_block_tokens-1
- for (int32_t i = 1; i < n_block_tokens; ++i) {
- common_sampler_sample(smpl, ctx_dft, beg + i, true);
+ if (is_dspark) {
+ // DSpark predicts the next token from position 0 and optionally truncates
+ // at the first position below the confidence threshold.
+ const float * conf = params.p_min > 0.0f ? llama_get_embeddings_nextn(ctx_dft) : nullptr;
- const auto * cur_p = common_sampler_get_candidates(smpl, true);
+ for (int32_t i = 0; i < n_block_tokens; ++i) {
+ const int32_t idx = beg + i;
- for (int k = 0; k < std::min(3, (int) cur_p->size); ++k) {
- LOG_DBG(" - seq_id %d, draft candidate %3d, pos %3d: %6d (%8.3f) '%s'\n",
- seq_id, k, i - 1, cur_p->data[k].id, cur_p->data[k].p,
- common_token_to_piece(ctx_dft, cur_p->data[k].id).c_str());
- }
+ if (conf && conf[(size_t) idx * n_embd_dec] < params.p_min) {
+ break;
+ }
- const llama_token id = cur_p->data[0].id;
+ common_sampler_sample(smpl, ctx_dft, idx, true);
- if (cur_p->data[0].p < params.p_min) {
- break;
+ const auto * cur_p = common_sampler_get_candidates(smpl, true);
+
+ for (int k = 0; k < std::min(3, (int) cur_p->size); ++k) {
+ LOG_DBG(" - seq_id %d, draft candidate %3d, pos %3d: %6d (%8.3f) '%s'\n",
+ seq_id, k, i, cur_p->data[k].id, cur_p->data[k].p,
+ common_token_to_piece(ctx_dft, cur_p->data[k].id).c_str());
+ }
+
+ const llama_token id = cur_p->data[0].id;
+
+ common_sampler_accept(smpl, id, true);
+
+ result.push_back(id);
}
+ } else {
+ // greedily read the predicted block at this sequence's noise positions 1..n_block_tokens-1
+ for (int32_t i = 1; i < n_block_tokens; ++i) {
+ common_sampler_sample(smpl, ctx_dft, beg + i, true);
- common_sampler_accept(smpl, id, true);
+ const auto * cur_p = common_sampler_get_candidates(smpl, true);
- result.push_back(id);
+ for (int k = 0; k < std::min(3, (int) cur_p->size); ++k) {
+ LOG_DBG(" - seq_id %d, draft candidate %3d, pos %3d: %6d (%8.3f) '%s'\n",
+ seq_id, k, i - 1, cur_p->data[k].id, cur_p->data[k].p,
+ common_token_to_piece(ctx_dft, cur_p->data[k].id).c_str());
+ }
+
+ const llama_token id = cur_p->data[0].id;
+
+ if (cur_p->data[0].p < params.p_min) {
+ break;
+ }
+
+ common_sampler_accept(smpl, id, true);
+
+ result.push_back(id);
+ }
}
if (result.size() < (size_t) params.n_min) {
@@ -2155,6 +2190,7 @@ std::string common_speculative_type_to_str(common_speculative_type type) {
case COMMON_SPECULATIVE_TYPE_DRAFT_EAGLE3: return "draft-eagle3";
case COMMON_SPECULATIVE_TYPE_DRAFT_MTP: return "draft-mtp";
case COMMON_SPECULATIVE_TYPE_DRAFT_DFLASH: return "draft-dflash";
+ case COMMON_SPECULATIVE_TYPE_DRAFT_DSPARK: return "draft-dspark";
case COMMON_SPECULATIVE_TYPE_NGRAM_SIMPLE: return "ngram-simple";
case COMMON_SPECULATIVE_TYPE_NGRAM_MAP_K: return "ngram-map-k";
case COMMON_SPECULATIVE_TYPE_NGRAM_MAP_K4V: return "ngram-map-k4v";
@@ -2208,6 +2244,7 @@ int32_t common_speculative_n_max(const common_params_speculative * spec) {
case COMMON_SPECULATIVE_TYPE_DRAFT_EAGLE3:
case COMMON_SPECULATIVE_TYPE_DRAFT_MTP:
case COMMON_SPECULATIVE_TYPE_DRAFT_DFLASH:
+ case COMMON_SPECULATIVE_TYPE_DRAFT_DSPARK:
n_max = std::max(n_max, std::max(0, spec->draft.n_max));
break;
case COMMON_SPECULATIVE_TYPE_NGRAM_SIMPLE:
@@ -2352,6 +2389,7 @@ common_speculative * common_speculative_init(common_params_speculative & params,
bool has_draft_eagle3 = (enabled_configs & (1u << COMMON_SPECULATIVE_TYPE_DRAFT_EAGLE3)) && params.draft.ctx_dft != nullptr;
bool has_draft_mtp = (enabled_configs & (1u << COMMON_SPECULATIVE_TYPE_DRAFT_MTP)) && params.draft.ctx_dft != nullptr;
bool has_draft_dflash = (enabled_configs & (1u << COMMON_SPECULATIVE_TYPE_DRAFT_DFLASH)) && params.draft.ctx_dft != nullptr;
+ bool has_draft_dspark = (enabled_configs & (1u << COMMON_SPECULATIVE_TYPE_DRAFT_DSPARK)) && params.draft.ctx_dft != nullptr;
@@ -2362,7 +2400,7 @@ common_speculative * common_speculative_init(common_params_speculative & params,
bool has_ngram_mod = (enabled_configs & (1u << COMMON_SPECULATIVE_TYPE_NGRAM_MOD));
// when adding a new type - update here the logic above
- static_assert(COMMON_SPECULATIVE_TYPE_COUNT == 10);
+ static_assert(COMMON_SPECULATIVE_TYPE_COUNT == 11);
// this list here defines the priority of the speculators
// the one with highest priority are listed first
@@ -2395,6 +2433,9 @@ common_speculative * common_speculative_init(common_params_speculative & params,
if (has_draft_dflash) {
configs.push_back(common_speculative_config(COMMON_SPECULATIVE_TYPE_DRAFT_DFLASH, params));
}
+ if (has_draft_dspark) {
+ configs.push_back(common_speculative_config(COMMON_SPECULATIVE_TYPE_DRAFT_DSPARK, params));
+ }
}
std::vector<std::unique_ptr<common_speculative_impl>> impls = {};
@@ -2419,6 +2460,11 @@ common_speculative * common_speculative_init(common_params_speculative & params,
impls.push_back(std::make_unique<common_speculative_impl_draft_dflash>(config.params, n_seq));
break;
}
+ case COMMON_SPECULATIVE_TYPE_DRAFT_DSPARK: {
+ impls.push_back(std::make_unique<common_speculative_impl_draft_dflash>(
+ config.params, n_seq, COMMON_SPECULATIVE_TYPE_DRAFT_DSPARK));
+ break;
+ }
case COMMON_SPECULATIVE_TYPE_NGRAM_SIMPLE: {
common_ngram_map ngram_map = get_common_ngram_map(config.type, config.params.ngram_simple);
diff --git a/conversion/__init__.py b/conversion/__init__.py
index 45c001b78f..1a47b851a0 100644
--- a/conversion/__init__.py
+++ b/conversion/__init__.py
@@ -53,6 +53,7 @@ TEXT_MODEL_MAP: dict[str, str] = {
"DeepseekV3ForCausalLM": "deepseek",
"DeepseekV32ForCausalLM": "deepseek",
"DFlashDraftModel": "qwen",
+ "Qwen3DSparkModel": "qwen",
"DeepseekV4ForCausalLM": "deepseek",
"DistilBertForMaskedLM": "bert",
"DistilBertForSequenceClassification": "bert",
diff --git a/conversion/qwen.py b/conversion/qwen.py
index 9bc2b99fde..d1127f7431 100644
--- a/conversion/qwen.py
+++ b/conversion/qwen.py
@@ -688,3 +688,23 @@ class DFlashModel(Qwen3Model):
if not name.startswith("model."):
name = "model." + name
return super().filter_tensors((name, gen))
+
+
[email protected]("Qwen3DSparkModel")
+class DSparkModel(DFlashModel):
+ # DSpark = DFlash + a semi-autoregressive Markov head
+ model_arch = gguf.MODEL_ARCH.DFLASH
+
+ def __init__(self, *args, **kwargs):
+ super().__init__(*args, **kwargs)
+ # normalize the flat DeepSpec schema to DFlash's nested dflash_config
+ self.hparams.setdefault("dflash_config", {
+ k: self.hparams[k] for k in ("target_layer_ids", "mask_token_id") if k in self.hparams
+ })
+
+ @classmethod
+ def filter_tensors(cls, item: tuple[str, Callable[[], Tensor]]) -> tuple[str, Callable[[], Tensor]] | None:
+ name, gen = item
+ if name.endswith(("embed_tokens.weight", "lm_head.weight")):
+ return None
+ return super().filter_tensors((name, gen))
diff --git a/docs/speculative.md b/docs/speculative.md
index 4100b92f8f..3957db85c9 100644
--- a/docs/speculative.md
+++ b/docs/speculative.md
@@ -78,6 +78,38 @@ See:
- #22105
+### DSpark (`draft-dspark`)
+
+DSpark extends DFlash with a semi-autoregressive _Markov head_: the draft still emits a whole
+block per forward pass, but each block position's logits are biased by a low-rank term keyed on
+the previous token, chained in-graph across the block. This keeps drafting at one decode per
+block while recovering some of the left-to-right signal that pure block diffusion loses.
+
+The draft is a small DeepSpec checkpoint trained for a specific target (for example
+[`deepseek-ai/dspark_qwen3_4b_block7`](https://huggingface.co/deepseek-ai/dspark_qwen3_4b_block7)
+for `Qwen/Qwen3-4B`). Convert it with `--target-model-dir` so it inherits the target's tokenizer
+and token embeddings:
+
+```bash
+python convert_hf_to_gguf.py deepseek-ai/dspark_qwen3_4b_block7 \
+ --target-model-dir Qwen/Qwen3-4B --outtype bf16 --outfile Qwen3-4B-DSpark.gguf
+
+llama-server -m Qwen3-4B.gguf -md Qwen3-4B-DSpark.gguf \
+ --spec-type draft-dspark --spec-draft-n-max 7 -fa on --jinja
+```
+
+`--spec-draft-n-max` is clamped to the draft model's trained block size.
+
+`--spec-draft-conf-min P` truncates each drafted block at the first position whose predicted
+acceptance (from the draft's confidence head, if present) falls below `P` (default 0 = disabled).
+
+Currently only drafts with a Qwen3 backbone are supported; support for other backbones
+(e.g. Gemma4) is planned.
+
+See:
+
+- #25173
+
### n-gram Cache (`ngram-cache`)
An n-gram is a sequence of n tokens. The n-gram cache implementation maintains statistics about short n-gram sequences.
@@ -173,7 +205,7 @@ If a draft model is combined with a draftless decoding the draftless decoding ha
### General Speculative Parameters
```
---spec-type [none|draft-simple|draft-eagle3|draft-dflash|draft-mtp|ngram-cache|ngram-simple|ngram-map-k|ngram-map-k4v|ngram-mod]
+--spec-type [none|draft-simple|draft-eagle3|draft-dflash|draft-dspark|draft-mtp|ngram-cache|ngram-simple|ngram-map-k|ngram-map-k4v|ngram-mod]
comma-separated list of types of speculative decoding to use
(default: none)
(env: LLAMA_ARG_SPEC_TYPE)
@@ -314,6 +346,7 @@ Specifies a comma-separated list of speculative decoding types to use.
| `draft-simple` | Use a simple draft model for speculation |
| `draft-eagle3` | Use an EAGLE-3 draft model that reads the target's hidden states |
| `draft-dflash` | Use a DFlash block-diffusion draft model that emits a block per step |
+| `draft-dspark` | Use a DSpark draft model (DFlash backbone + semi-autoregressive Markov head) |
| `draft-mtp` | Use Multi Token Prediction (MTP) heads from the main model |
| `ngram-cache` | Use n-gram cache lookup |
| `ngram-simple` | Use simple n-gram pattern matching |
diff --git a/gguf-py/gguf/constants.py b/gguf-py/gguf/constants.py
index 4df94b9a3a..2ebefaa3b4 100644
--- a/gguf-py/gguf/constants.py
+++ b/gguf-py/gguf/constants.py
@@ -988,6 +988,10 @@ class MODEL_TENSOR(IntEnum):
# eagle3
FC = auto() # feature fusion layer
D2T = auto() # draft to target vocabulary mapping
+ # dspark
+ DSPARK_MARKOV_W1 = auto() # markov head: prev-token embed
+ DSPARK_MARKOV_W2 = auto() # markov head: bias projection
+ DSPARK_CONF_PROJ = auto() # confidence head
# lfm2 audio
A_ENC_NORM_CONV = auto()
A_ENC_LINEAR_POS = auto()
@@ -1617,6 +1621,9 @@ TENSOR_NAMES: dict[MODEL_TENSOR, str] = {
MODEL_TENSOR.NEXTN_SHARED_HEAD_HEAD: "blk.{bid}.nextn.shared_head_head",
MODEL_TENSOR.NEXTN_SHARED_HEAD_NORM: "blk.{bid}.nextn.shared_head_norm",
MODEL_TENSOR.FC: "fc",
+ MODEL_TENSOR.DSPARK_MARKOV_W1: "markov_w1",
+ MODEL_TENSOR.DSPARK_MARKOV_W2: "markov_w2",
+ MODEL_TENSOR.DSPARK_CONF_PROJ: "conf_proj",
MODEL_TENSOR.D2T: "d2t",
}
@@ -4362,6 +4369,10 @@ MODEL_TENSORS: dict[MODEL_ARCH, list[MODEL_TENSOR]] = {
MODEL_TENSOR.FFN_UP,
MODEL_TENSOR.FC,
MODEL_TENSOR.ENC_OUTPUT_NORM,
+ # optional DSpark heads
+ MODEL_TENSOR.DSPARK_MARKOV_W1,
+ MODEL_TENSOR.DSPARK_MARKOV_W2,
+ MODEL_TENSOR.DSPARK_CONF_PROJ,
],
MODEL_ARCH.MISTRAL4: [
MODEL_TENSOR.TOKEN_EMBD,
diff --git a/gguf-py/gguf/tensor_mapping.py b/gguf-py/gguf/tensor_mapping.py
index 8299ac25b4..5562d43277 100644
--- a/gguf-py/gguf/tensor_mapping.py
+++ b/gguf-py/gguf/tensor_mapping.py
@@ -1304,6 +1304,18 @@ class TensorNameMap:
"model.fc", # dflash
),
+ MODEL_TENSOR.DSPARK_MARKOV_W1: (
+ "model.markov_head.markov_w1", # dspark
+ ),
+
+ MODEL_TENSOR.DSPARK_MARKOV_W2: (
+ "model.markov_head.markov_w2", # dspark
+ ),
+
+ MODEL_TENSOR.DSPARK_CONF_PROJ: (
+ "model.confidence_head.proj", # dspark
+ ),
+
MODEL_TENSOR.CLS: (
"classifier", # jina
"classifier.dense", # roberta
diff --git a/src/llama-arch.cpp b/src/llama-arch.cpp
index a0945e501d..e81ff647ee 100644
--- a/src/llama-arch.cpp
+++ b/src/llama-arch.cpp
@@ -616,6 +616,9 @@ static const std::map<llm_tensor, const char *> LLM_TENSOR_NAMES = {
{ LLM_TENSOR_MASKED_EMBD_ORDERING, "masked_embd_ordering" },
{ LLM_TENSOR_FC, "fc" },
{ LLM_TENSOR_D2T, "d2t" },
+ { LLM_TENSOR_DSPARK_MARKOV_W1, "markov_w1" },
+ { LLM_TENSOR_DSPARK_MARKOV_W2, "markov_w2" },
+ { LLM_TENSOR_DSPARK_CONF_PROJ, "conf_proj" },
};
// declare information about the model weight tensors:
@@ -870,6 +873,10 @@ static const std::map<llm_tensor, llm_tensor_info> LLM_TENSOR_INFOS = {
// eagle3
{LLM_TENSOR_FC, {LLM_TENSOR_LAYER_OUTPUT, GGML_OP_MUL_MAT}},
{LLM_TENSOR_D2T, {LLM_TENSOR_LAYER_OUTPUT, GGML_OP_GET_ROWS}},
+ // dspark
+ {LLM_TENSOR_DSPARK_MARKOV_W1, {LLM_TENSOR_LAYER_OUTPUT, GGML_OP_GET_ROWS}},
+ {LLM_TENSOR_DSPARK_MARKOV_W2, {LLM_TENSOR_LAYER_OUTPUT, GGML_OP_MUL_MAT}},
+ {LLM_TENSOR_DSPARK_CONF_PROJ, {LLM_TENSOR_LAYER_OUTPUT, GGML_OP_MUL_MAT}},
};
LLM_KV::LLM_KV(llm_arch arch, const char * suffix) : arch(arch), suffix(suffix) {}
diff --git a/src/llama-arch.h b/src/llama-arch.h
index fb5eb3920b..cbc97085ea 100644
--- a/src/llama-arch.h
+++ b/src/llama-arch.h
@@ -624,6 +624,9 @@ enum llm_tensor {
LLM_TENSOR_MASKED_EMBD_ORDERING,
LLM_TENSOR_FC,
LLM_TENSOR_D2T,
+ LLM_TENSOR_DSPARK_MARKOV_W1,
+ LLM_TENSOR_DSPARK_MARKOV_W2,
+ LLM_TENSOR_DSPARK_CONF_PROJ,
};
diff --git a/src/llama-model.h b/src/llama-model.h
index 36d0480e5e..d6a40fa302 100644
--- a/src/llama-model.h
+++ b/src/llama-model.h
@@ -606,6 +606,12 @@ struct llama_model {
struct ggml_tensor * fc = nullptr; // feature fusion layer
struct ggml_tensor * d2t = nullptr; // draft to target vocabulary mapping
+ // dspark
+ struct ggml_tensor * dspark_markov_w1 = nullptr;
+ struct ggml_tensor * dspark_markov_w2 = nullptr;
+ struct ggml_tensor * dspark_conf_proj = nullptr;
+ struct ggml_tensor * dspark_conf_proj_b = nullptr;
+
// unified vector to store target-model extracted layer ids in eagle3, dflash, etc.
std::vector<int32_t> target_layer_ids;
diff --git a/src/models/dflash.cpp b/src/models/dflash.cpp
index 427eed4594..dcff3aec9f 100644
--- a/src/models/dflash.cpp
+++ b/src/models/dflash.cpp
@@ -37,6 +37,23 @@ void llama_model_dflash::load_arch_tensors(llama_model_loader &) {
const int64_t n_embd_inp = hparams.n_embd_inp_enc();
+ // DSpark = DFlash + a semi-autoregressive Markov head and Confidence head
+ //
+ // TODO: only Qwen3-style backbones are supported for now; other backbones (e.g. Gemma4)
+ // need their own conversion path and graph tweaks
+ const struct ggml_tensor * markov_meta = ml->get_tensor_meta("markov_w1.weight");
+ if (markov_meta) {
+ const int64_t dspark_markov_rank = markov_meta->ne[0];
+
+ dspark_markov_w1 = create_tensor(tn(LLM_TENSOR_DSPARK_MARKOV_W1, "weight"), { dspark_markov_rank, n_vocab }, 0);
+ dspark_markov_w2 = create_tensor(tn(LLM_TENSOR_DSPARK_MARKOV_W2, "weight"), { dspark_markov_rank, n_vocab }, 0);
+
+ dspark_conf_proj = create_tensor(tn(LLM_TENSOR_DSPARK_CONF_PROJ, "weight"), { n_embd + dspark_markov_rank, 1 }, 0);
+ dspark_conf_proj_b = create_tensor(tn(LLM_TENSOR_DSPARK_CONF_PROJ, "bias"), { 1 }, TENSOR_NOT_REQUIRED);
+
+ LLAMA_LOG_INFO("%s: DFlash with DSpark markov head (rank = %lld)\n", __func__, (long long) dspark_markov_rank);
+ }
+
fc = create_tensor(tn(LLM_TENSOR_FC, "weight"), { n_embd_inp, n_embd }, 0);
output_norm_enc = create_tensor(tn(LLM_TENSOR_ENC_OUTPUT_NORM, "weight"), { n_embd }, 0); // encoder hidden_norm (after fc)
output_norm = create_tensor(tn(LLM_TENSOR_OUTPUT_NORM, "weight"), { n_embd }, 0); // decoder final norm
@@ -105,6 +122,94 @@ llama_model_dflash::graph<true>::graph(const llama_model & model, const llm_grap
ggml_build_forward_expand(gf, cur);
}
+// DSpark (DFlash + Markov & Confidence head): Markov bias on the draft logits, chained per block position
+static void build_dspark_markov_head(llm_graph_context & g, const llama_model & model, ggml_tensor * tokens) {
+ ggml_context * ctx0 = g.ctx0;
+ auto & res = g.res;
+
+ ggml_tensor * w1 = model.dspark_markov_w1;
+ ggml_tensor * w2 = model.dspark_markov_w2;
+ GGML_ASSERT(w1 && w2 && model.dspark_conf_proj && "DSpark markov/confidence weights not loaded");
+
+ ggml_tensor * base = res->t_logits; // [n_vocab, n_tokens]
+ const int64_t n_vocab = base->ne[0];
+ const int64_t n_tok = base->ne[1];
+
+ const auto it = model.gguf_kv.find("dflash.block_size");
+ GGML_ASSERT(it != model.gguf_kv.end() && "DSpark draft requires 'dflash.block_size' in GGUF metadata");
+ const int64_t block_size = std::stoi(it->second);
+ GGML_ASSERT(block_size > 0);
+
+ const int64_t n_blocks = g.ubatch.n_seqs_unq;
+ GGML_ASSERT(n_blocks > 0 && n_tok % n_blocks == 0 && "DSpark markov head requires equal-size blocks");
+ // runtime tokens per block in this ubatch (anchor + drafted positions), bounded by training block_size
+ const int64_t block_drafts = n_tok / n_blocks;
+ if (block_drafts > block_size) {
+ return;
+ }
+
+ // anchor (committed last) token of every block: token 0 of each block, i.e. a strided view
+ const size_t token_stride = (size_t) block_drafts * tokens->nb[0];
+ const size_t base_stride = (size_t) block_drafts * base->nb[1];
+
+ ggml_tensor * prev = ggml_view_2d(ctx0, tokens, 1, n_blocks, token_stride, 0);
+ prev = ggml_cont_1d(ctx0, prev, n_blocks);
+
+ // confidence head input: predicts per-position acceptance
+ ggml_tensor * conf_inp = res->t_embd; // [n_embd, n_tok]
+
+ ggml_tensor * cat = nullptr;
+ ggml_tensor * cat_conf = nullptr;
+
+ // TODO: the in-graph chain is greedy (argmax); sampling params affect only the final
+ // token pick, not the Markov conditioning path
+ for (int64_t i = 0; i < block_drafts; ++i) {
+ ggml_tensor * w1_prev = ggml_get_rows(ctx0, w1, prev); // [R, n_blocks]
+ ggml_tensor * bias = ggml_mul_mat(ctx0, w2, w1_prev); // [n_vocab, n_blocks]
+
+ // position i of every block: strided view [n_vocab, n_blocks]
+ ggml_tensor * base_i = ggml_view_2d(ctx0, base, n_vocab, n_blocks, base_stride, i*base->nb[1]);
+ ggml_tensor * col = ggml_add(ctx0, base_i, bias);
+
+ cat = cat ? ggml_concat(ctx0, cat, col, 1) : col;
+
+ // conf(i) = sigmoid(conf_proj . [conf_inp(i); markov_w1[prev(i)]] + b) -- [1, n_blocks]
+ ggml_tensor * conf_inp_i = ggml_view_2d(ctx0, conf_inp, conf_inp->ne[0], n_blocks,
+ (size_t) block_drafts * conf_inp->nb[1], i*conf_inp->nb[1]);
+ ggml_tensor * feat = ggml_concat(ctx0, ggml_cont(ctx0, conf_inp_i), w1_prev, 0);
+ ggml_tensor * conf = ggml_mul_mat(ctx0, model.dspark_conf_proj, feat);
+ if (model.dspark_conf_proj_b) {
+ conf = ggml_add(ctx0, conf, model.dspark_conf_proj_b);
+ }
+ conf = ggml_sigmoid(ctx0, conf);
+
+ cat_conf = cat_conf ? ggml_concat(ctx0, cat_conf, conf, 1) : conf;
+
+ if (i + 1 < block_drafts) {
+ prev = ggml_argmax(ctx0, col);
+ }
+ }
+
+ // cat is position-major; restore ubatch block-major order
+ ggml_tensor * out = ggml_reshape_3d(ctx0, cat, n_vocab, n_blocks, block_drafts);
+ out = ggml_cont(ctx0, ggml_permute(ctx0, out, 0, 2, 1, 3)); // [n_vocab, block_drafts, n_blocks]
+ out = ggml_reshape_2d(ctx0, out, n_vocab, n_tok);
+
+ {
+ ggml_tensor * conf = ggml_reshape_3d(ctx0, cat_conf, 1, n_blocks, block_drafts);
+ conf = ggml_cont(ctx0, ggml_permute(ctx0, conf, 0, 2, 1, 3));
+ conf = ggml_reshape_2d(ctx0, conf, 1, n_tok);
+
+ // note: broadcast the [1, n_tok] confidences to n_embd-wide rows to be able to reuse `llama_get_embeddings_nextn`
+ conf = ggml_repeat(ctx0, conf, res->t_embd);
+ res->t_h_nextn = conf;
+ ggml_build_forward_expand(g.gf, conf);
+ }
+
+ res->t_logits = out;
+ ggml_build_forward_expand(g.gf, out);
+}
+
// DFlash decoder, dual-mode by batch type:
// * embd batch -> fused target features: project + inject K/V into the cache.
// * token batch -> noise-block diffusion: attend over [committed, MASK...] to generate draft tokens
@@ -210,6 +315,8 @@ llama_model_dflash::graph<false>::graph(const llama_model & model, const llm_gra
inp->tokens = ggml_new_tensor_1d(ctx0, GGML_TYPE_I32, n_tokens);
ggml_set_input(inp->tokens);
+ ggml_tensor * inp_tokens = inp->tokens;
+
ggml_tensor * inpL = ggml_get_rows(ctx0, tok_embd, inp->tokens);
cb(inpL, "inp_noise_embd", -1);
@@ -290,4 +397,9 @@ llama_model_dflash::graph<false>::graph(const llama_model & model, const llm_gra
res->t_logits = cur;
ggml_build_forward_expand(gf, cur);
+
+ // DSpark: bias the draft logits with the Markov head
+ if (model.dspark_markov_w1) {
+ build_dspark_markov_head(*this, model, inp_tokens);
+ }
}
diff --git a/tools/cli/README.md b/tools/cli/README.md
index 972ea04dc7..bcddd05702 100644
--- a/tools/cli/README.md
+++ b/tools/cli/README.md
@@ -203,7 +203,7 @@
| `--spec-draft-device, -devd, --device-draft <dev1,dev2,..>` | comma-separated list of devices to use for offloading the draft model (none = don't offload)<br/>use --list-devices to see a list of available devices |
| `--spec-draft-ngl, -ngld, --gpu-layers-draft, --n-gpu-layers-draft N` | max. number of draft model layers to store in VRAM, either an exact number, 'auto', or 'all' (default: auto)<br/>(env: LLAMA_ARG_N_GPU_LAYERS_DRAFT) |
| `--spec-draft-model, -md, --model-draft FNAME` | draft model for speculative decoding (default: unused)<br/>(env: LLAMA_ARG_SPEC_DRAFT_MODEL) |
-| `--spec-type none,draft-simple,draft-eagle3,draft-mtp,draft-dflash,ngram-simple,ngram-map-k,ngram-map-k4v,ngram-mod,ngram-cache` | comma-separated list of types of speculative decoding to use (default: none)<br/><br/>(env: LLAMA_ARG_SPEC_TYPE) |
+| `--spec-type none,draft-simple,draft-eagle3,draft-mtp,draft-dflash,draft-dspark,ngram-simple,ngram-map-k,ngram-map-k4v,ngram-mod,ngram-cache` | comma-separated list of types of speculative decoding to use (default: none)<br/><br/>(env: LLAMA_ARG_SPEC_TYPE) |
| `--spec-ngram-mod-n-min N` | minimum number of ngram tokens to use for ngram-based speculative decoding (default: 48) |
| `--spec-ngram-mod-n-max N` | maximum number of ngram tokens to use for ngram-based speculative decoding (default: 64) |
| `--spec-ngram-mod-n-match N` | ngram-mod lookup length (default: 24) |
diff --git a/tools/server/README.md b/tools/server/README.md
index 25aacf9f51..f45c018972 100644
--- a/tools/server/README.md
+++ b/tools/server/README.md
@@ -259,7 +259,7 @@ For the full list of features, please refer to [server's changelog](https://gith
| `--spec-draft-device, -devd, --device-draft <dev1,dev2,..>` | comma-separated list of devices to use for offloading the draft model (none = don't offload)<br/>use --list-devices to see a list of available devices |
| `--spec-draft-ngl, -ngld, --gpu-layers-draft, --n-gpu-layers-draft N` | max. number of draft model layers to store in VRAM, either an exact number, 'auto', or 'all' (default: auto)<br/>(env: LLAMA_ARG_N_GPU_LAYERS_DRAFT) |
| `--spec-draft-model, -md, --model-draft FNAME` | draft model for speculative decoding (default: unused)<br/>(env: LLAMA_ARG_SPEC_DRAFT_MODEL) |
-| `--spec-type none,draft-simple,draft-eagle3,draft-mtp,draft-dflash,ngram-simple,ngram-map-k,ngram-map-k4v,ngram-mod,ngram-cache` | comma-separated list of types of speculative decoding to use (default: none)<br/><br/>(env: LLAMA_ARG_SPEC_TYPE) |
+| `--spec-type none,draft-simple,draft-eagle3,draft-mtp,draft-dflash,draft-dspark,ngram-simple,ngram-map-k,ngram-map-k4v,ngram-mod,ngram-cache` | comma-separated list of types of speculative decoding to use (default: none)<br/><br/>(env: LLAMA_ARG_SPEC_TYPE) |
| `--spec-ngram-mod-n-min N` | minimum number of ngram tokens to use for ngram-based speculative decoding (default: 48) |
| `--spec-ngram-mod-n-max N` | maximum number of ngram tokens to use for ngram-based speculative decoding (default: 64) |
| `--spec-ngram-mod-n-match N` | ngram-mod lookup length (default: 24) |
diff --git a/tools/server/server-schema.cpp b/tools/server/server-schema.cpp
index e880f4ca72..674d3ba337 100644
--- a/tools/server/server-schema.cpp
+++ b/tools/server/server-schema.cpp
@@ -209,6 +209,7 @@ std::vector<std::unique_ptr<field>> make_llama_cmpl_schema(const common_params &
->set_hard_limits(0.0f, 1.0f)
->set_desc("Minimum speculative decoding probability for draft tokens (0 = greedy)"));
+
add((new field_str("speculative.type"))
->set_desc("Speculative decoding method (for debugging and research purposes)")
->set_handler([&](field_eval_context & ctx, const json & data) {