LLM inference in C/C++
C++ ggml
spec : add DFlash2 support (local convolution + candidate selector) (#27342) (#27816)
Xuan-Son Nguyenb10f9ca58c89
diff --git a/common/speculative.cpp b/common/speculative.cppindex 393a73bc39..b5348ab6f3 100644--- a/common/speculative.cpp+++ b/common/speculative.cpp@@ -925,6 +925,10 @@ struct common_speculative_impl_draft_dflash : public common_speculative_impl {int32_t block_size = 0;llama_token mask_token_id = 0;+ bool is_dflash2 = false;+ bool is_mrope = false;+ int32_t selector_top_k = 0;+// draft-dspark: the draft carries a Markov head and uses an anchor-first block layoutconst bool is_dspark;@@ -969,6 +973,9 @@ struct common_speculative_impl_draft_dflash : public common_speculative_impl {sample_from_anchor = std::strcmp(buf, "true") == 0;}}++ selector_top_k = llama_model_dflash_selector_top_k(model_dft);+ is_dflash2 = selector_top_k > 0;mask_token_id = llama_vocab_mask(llama_model_get_vocab(model_dft));LOG_INF("%s: adding speculative implementation '%s'\n", __func__, common_speculative_type_to_str(type).c_str());@@ -990,6 +997,13 @@ struct common_speculative_impl_draft_dflash : public common_speculative_impl {batch = llama_batch_init(llama_n_batch(ctx_dft), 0, n_seq);batch_inject = llama_batch_init(llama_n_batch(ctx_dft), n_embd_dec, n_seq);+ // embd batches on an M-RoPE draft need 4 position rows per token+ is_mrope = llama_model_rope_type(model_dft) == LLAMA_ROPE_TYPE_MROPE;+ if (is_mrope) {+ free(batch_inject.pos);+ batch_inject.pos = (llama_pos *) malloc(sizeof(llama_pos) * 4 * llama_n_batch(ctx_dft));+ }+smpls.resize(n_seq);for (auto & s : smpls) {common_params_sampling sparams;@@ -1001,7 +1015,7 @@ struct common_speculative_impl_draft_dflash : public common_speculative_impl {// offload draft sampling to the backendbackend_chains.assign(n_seq, nullptr);- if (this->params.backend_sampling) {+ if (this->params.backend_sampling && !is_dflash2) {for (llama_seq_id seq_id = 0; seq_id < (llama_seq_id) n_seq; ++seq_id) {llama_sampler * chain = llama_sampler_chain_init(llama_sampler_chain_default_params());llama_sampler_chain_add(chain, llama_sampler_init_top_k(10));@@ -1020,7 +1034,8 @@ struct common_speculative_impl_draft_dflash : public common_speculative_impl {llama_set_embeddings_layer_inp(ctx_tgt, (uint32_t) target_layer_ids[k], true);}- llama_set_embeddings_nextn(ctx_dft, true, /*masked*/ true);+ // DFlash2 reads its selector lattice from h_nextn and never consumes raw logits.+ llama_set_embeddings_nextn(ctx_dft, true, /*masked*/ !is_dflash2);llama_set_causal_attn(ctx_dft, false); // DFlash needs non-causal attention}@@ -1121,11 +1136,24 @@ struct common_speculative_impl_draft_dflash : public common_speculative_impl {}// fuse extracted features through DFlash encoder+ // M-RoPE drafts read 4 position rows per token from embd batches, so pass them explicitly+ std::vector<llama_pos> enc_pos;+ if (is_mrope) {+ enc_pos.resize((size_t) 4 * n_chunk);+ for (int32_t i = 0; i < n_chunk; ++i) {+ const llama_pos p = batch_in.pos[i_batch_beg[seq_id] + offset + i];+ enc_pos[0 * n_chunk + i] = p;+ enc_pos[1 * n_chunk + i] = p;+ enc_pos[2 * n_chunk + i] = p;+ enc_pos[3 * n_chunk + i] = 0;+ }+ }+llama_batch enc_batch = {/*.n_tokens =*/ n_chunk,/*.token =*/ nullptr,/*.embd =*/ features_buf.data(),- /*.pos =*/ nullptr,+ /*.pos =*/ is_mrope ? enc_pos.data() : nullptr,/*.n_seq_id =*/ nullptr,/*.seq_id =*/ nullptr,/*.logits =*/ nullptr,@@ -1146,7 +1174,13 @@ struct common_speculative_impl_draft_dflash : public common_speculative_impl {std::memcpy(batch_inject.embd, inp_g, (size_t) n_chunk * n_embd_dec * sizeof(float));for (int32_t i = 0; i < n_chunk; ++i) {- batch_inject.pos[i] = batch_in.pos[i_batch_beg[seq_id] + offset + i];+ const llama_pos p = batch_in.pos[i_batch_beg[seq_id] + offset + i];+ batch_inject.pos[i] = p;+ if (is_mrope) {+ batch_inject.pos[1 * n_chunk + i] = p;+ batch_inject.pos[2 * n_chunk + i] = p;+ batch_inject.pos[3 * n_chunk + i] = 0;+ }batch_inject.n_seq_id[i] = 1;batch_inject.seq_id[i][0] = seq_id;batch_inject.logits[i] = false;@@ -1189,7 +1223,7 @@ struct common_speculative_impl_draft_dflash : public common_speculative_impl {i_block_beg[seq_id] = batch.n_tokens;n_block [seq_id] = n_block_tokens;for (int32_t i = 0; i < n_block_tokens; ++i) {- common_batch_add(batch, i == 0 ? dp.id_last : mask_token_id, n + i, { seq_id }, true);+ common_batch_add(batch, i == 0 ? dp.id_last : mask_token_id, n + i, { seq_id }, !is_dflash2);}}@@ -1217,6 +1251,36 @@ struct common_speculative_impl_draft_dflash : public common_speculative_impl {auto & result = *dp.result;+ if (is_dflash2) {+ const float * lattice = llama_get_embeddings_nextn(ctx_dft);+ GGML_ASSERT(lattice && "DFlash2 selector produced no lattice");++ int32_t predecessor = 0;+ for (int32_t i = 1; i < n_block_tokens; ++i) {+ const float * row = lattice + (size_t) (beg + i) * n_embd_dec;+ const float * scores = row + selector_top_k + (size_t) predecessor * selector_top_k;++ predecessor = (int32_t) std::distance(scores,+ std::max_element(scores, scores + selector_top_k));+ if (params.p_min > 0.0f) {+ // softmax(scores) at the argmax, i.e. 1 / sum(exp(s_k - s_max))+ float sum = 0.0f;+ for (int32_t k = 0; k < selector_top_k; ++k) {+ sum += std::exp(scores[k] - scores[predecessor]);+ }+ if (1.0f / sum < params.p_min) {+ break;+ }+ }+ result.push_back((llama_token) row[predecessor]);+ }++ if (result.size() < (size_t) params.n_min) {+ result.clear();+ }+ continue;+ }+if (is_dspark) {// DSpark: read from the first draft slot, truncate below the confidence thresholdconst float * conf = params.p_min > 0.0f ? llama_get_embeddings_nextn(ctx_dft) : nullptr;diff --git a/conversion/__init__.py b/conversion/__init__.pyindex 8de97e9596..ab6cacbc78 100644--- a/conversion/__init__.py+++ b/conversion/__init__.py@@ -54,6 +54,7 @@ TEXT_MODEL_MAP: dict[str, str] = {"DeepseekV3ForCausalLM": "deepseek","DeepseekV32ForCausalLM": "deepseek","DFlashDraftModel": "qwen",+ "DFlash2DraftModel": "qwen","Qwen3DSparkModel": "qwen","DSparkDraftModel": "qwen","DSparkSpeculator": "qwen",diff --git a/conversion/qwen.py b/conversion/qwen.pyindex cdba8a63e9..c5297418c9 100644--- a/conversion/qwen.py+++ b/conversion/qwen.py@@ -639,7 +639,7 @@ class Qwen3_5MoeTextModel(_Qwen35MRopeMixin, _LinearAttentionVReorderBase):model_arch = gguf.MODEL_ARCH.QWEN35MOE[email protected]("DFlashDraftModel")[email protected]("DFlashDraftModel", "DFlash2DraftModel")@ModelBase.example("z-lab/Qwen3.5-9B-DFlash")class DFlashModel(Qwen3Model):model_arch = gguf.MODEL_ARCH.DFLASH@@ -678,9 +678,31 @@ class DFlashModel(Qwen3Model):def set_gguf_parameters(self):super().set_gguf_parameters()- block_size = self.hparams.get("block_size", 16)- self.gguf_writer.add_block_size(block_size)dflash_config = self.hparams.get("dflash_config", {})+ block_size = dflash_config.get("block_size", self.hparams.get("block_size", 16))+ self.gguf_writer.add_block_size(block_size)++ if "conv_kernel_size" in dflash_config:+ self.gguf_writer.add_conv_kernel_size(int(dflash_config["conv_kernel_size"]))+ self.gguf_writer.add_conv_group_size(int(dflash_config["conv_group_size"]))+ self.gguf_writer.add_selector_rank(int(dflash_config["selector_rank"]))+ self.gguf_writer.add_selector_top_k(int(dflash_config["selector_top_k"]))++ output_multiplier = dflash_config.get(+ "output_multiplier", self.hparams.get("output_multiplier")+ )+ if output_multiplier is not None:+ self.gguf_writer.add_logit_scale(float(output_multiplier))+ softcap = dflash_config.get(+ "final_logit_softcapping", self.hparams.get("final_logit_softcapping")+ )+ if softcap is not None and float(softcap) > 0:+ self.gguf_writer.add_final_logit_softcapping(float(softcap))+ embedding_scale = dflash_config.get(+ "input_embedding_scale", self.hparams.get("input_embedding_scale")+ )+ if embedding_scale is not None:+ self.gguf_writer.add_embedding_scale(float(embedding_scale))target_layer_ids = dflash_config.get("target_layer_ids", [])if target_layer_ids:@@ -695,6 +717,21 @@ class DFlashModel(Qwen3Model):self.gguf_writer.add_sliding_window(sliding_window)self.gguf_writer.add_sliding_window_pattern(is_swa)+ # M-RoPE target: the draft ropes on the temporal dim only, so write+ # degenerate sections [n_rot/2, 0, 0, 0]+ if self._target_uses_mrope():+ head_dim = self.hparams.get("head_dim") or self.hparams["hidden_size"] // self.hparams["num_attention_heads"]+ self.gguf_writer.add_rope_dimension_sections([head_dim // 2, 0, 0, 0])++ def _target_uses_mrope(self) -> bool:+ if self.target_model_dir is None:+ return False+ with open(self.target_model_dir / "config.json", "r", encoding="utf-8") as f:+ cfg = json.load(f)+ cfg = cfg.get("text_config", cfg)+ rope = cfg.get("rope_parameters") or cfg.get("rope_scaling") or {}+ return "mrope_section" in rope+@classmethoddef filter_tensors(cls, item: tuple[str, Callable[[], Tensor]]) -> tuple[str, Callable[[], Tensor]] | None:name, gen = item@@ -702,10 +739,29 @@ class DFlashModel(Qwen3Model):name = "model." + namereturn super().filter_tensors((name, gen))+ _ROPE_PERMUTE_SUFFIXES = (+ "self_attn.q_proj.weight",+ "self_attn.k_proj.weight",+ "self_attn.q_norm.weight",+ "self_attn.k_norm.weight",+ )+def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]:if name == "model.embed_tokens.weight" and not self.hparams.get("has_embed_tokens", True):return+ # interleaved-rope checkpoints (rope_is_neox_style = false) -> NeoX layout: per head, even dims first then odd+ if not self.hparams.get("rope_is_neox_style", True) and name.endswith(self._ROPE_PERMUTE_SUFFIXES):+ head_dim = self.hparams["head_dim"]+ shape = data_torch.shape+ data_torch = data_torch.reshape(-1, head_dim // 2, 2, *shape[1:]).transpose(1, 2).reshape(shape)++ if name in (+ "model.candidate_selector.predecessor_codebook",+ "model.candidate_selector.successor_codebook",+ ):+ name += ".weight"+yield from super().modify_tensors(data_torch, name, bid)diff --git a/gguf-py/gguf/constants.py b/gguf-py/gguf/constants.pyindex f236a5d2c9..0a223831d5 100644--- a/gguf-py/gguf/constants.py+++ b/gguf-py/gguf/constants.py@@ -162,6 +162,10 @@ class Keys:TARGET_LAYERS = "{arch}.target_layers"TARGET_HIDDEN_SIZE = "{arch}.target_hidden_size"BLOCK_SIZE = "{arch}.block_size"+ CONV_KERNEL_SIZE = "{arch}.conv_kernel_size"+ CONV_GROUP_SIZE = "{arch}.conv_group_size"+ SELECTOR_RANK = "{arch}.selector_rank"+ SELECTOR_TOP_K = "{arch}.selector_top_k"SAMPLE_FROM_ANCHOR = "{arch}.sample_from_anchor"NORM_BEFORE_RESIDUAL = "{arch}.norm_before_residual"NORM_BEFORE_FC = "{arch}.norm_before_fc"@@ -1146,6 +1150,13 @@ class MODEL_TENSOR(IntEnum):DSPARK_MARKOV_W1 = auto() # markov head: prev-token embedDSPARK_MARKOV_W2 = auto() # markov head: bias projectionDSPARK_CONF_PROJ = auto() # confidence head+ DFLASH_ATTN_CONV_BASE = auto()+ DFLASH_ATTN_CONV_PROJ = auto()+ DFLASH_FFN_CONV_BASE = auto()+ DFLASH_FFN_CONV_PROJ = auto()+ DFLASH_SELECTOR_PREV = auto()+ DFLASH_SELECTOR_NEXT = auto()+ DFLASH_SELECTOR_HIDDEN = auto()# lfm2 audioA_ENC_NORM_CONV = auto()A_ENC_LINEAR_POS = auto()@@ -1895,6 +1906,13 @@ TENSOR_NAMES: dict[MODEL_TENSOR, str] = {MODEL_TENSOR.DSPARK_MARKOV_W1: "markov_w1",MODEL_TENSOR.DSPARK_MARKOV_W2: "markov_w2",MODEL_TENSOR.DSPARK_CONF_PROJ: "conf_proj",+ MODEL_TENSOR.DFLASH_ATTN_CONV_BASE: "blk.{bid}.attn_conv_base",+ MODEL_TENSOR.DFLASH_ATTN_CONV_PROJ: "blk.{bid}.attn_conv_proj",+ MODEL_TENSOR.DFLASH_FFN_CONV_BASE: "blk.{bid}.ffn_conv_base",+ MODEL_TENSOR.DFLASH_FFN_CONV_PROJ: "blk.{bid}.ffn_conv_proj",+ MODEL_TENSOR.DFLASH_SELECTOR_PREV: "selector_predecessor",+ MODEL_TENSOR.DFLASH_SELECTOR_NEXT: "selector_successor",+ MODEL_TENSOR.DFLASH_SELECTOR_HIDDEN: "selector_hidden",MODEL_TENSOR.D2T: "d2t",}@@ -4953,6 +4971,13 @@ MODEL_TENSORS: dict[MODEL_ARCH, list[MODEL_TENSOR]] = {MODEL_TENSOR.DSPARK_MARKOV_W1,MODEL_TENSOR.DSPARK_MARKOV_W2,MODEL_TENSOR.DSPARK_CONF_PROJ,+ MODEL_TENSOR.DFLASH_ATTN_CONV_BASE,+ MODEL_TENSOR.DFLASH_ATTN_CONV_PROJ,+ MODEL_TENSOR.DFLASH_FFN_CONV_BASE,+ MODEL_TENSOR.DFLASH_FFN_CONV_PROJ,+ MODEL_TENSOR.DFLASH_SELECTOR_PREV,+ MODEL_TENSOR.DFLASH_SELECTOR_NEXT,+ MODEL_TENSOR.DFLASH_SELECTOR_HIDDEN,],MODEL_ARCH.MISTRAL4: [MODEL_TENSOR.TOKEN_EMBD,diff --git a/gguf-py/gguf/gguf_writer.py b/gguf-py/gguf/gguf_writer.pyindex d8a96a27bd..fb6602a354 100644--- a/gguf-py/gguf/gguf_writer.py+++ b/gguf-py/gguf/gguf_writer.py@@ -993,6 +993,18 @@ class GGUFWriter:def add_block_size(self, value: int) -> None:self.add_uint32(Keys.LLM.BLOCK_SIZE.format(arch=self.arch), value)+ def add_conv_kernel_size(self, value: int) -> None:+ self.add_uint32(Keys.LLM.CONV_KERNEL_SIZE.format(arch=self.arch), value)++ def add_conv_group_size(self, value: int) -> None:+ self.add_uint32(Keys.LLM.CONV_GROUP_SIZE.format(arch=self.arch), value)++ def add_selector_rank(self, value: int) -> None:+ self.add_uint32(Keys.LLM.SELECTOR_RANK.format(arch=self.arch), value)++ def add_selector_top_k(self, value: int) -> None:+ self.add_uint32(Keys.LLM.SELECTOR_TOP_K.format(arch=self.arch), value)+def add_sample_from_anchor(self, value: bool) -> None:self.add_bool(Keys.LLM.SAMPLE_FROM_ANCHOR.format(arch=self.arch), value)diff --git a/gguf-py/gguf/tensor_mapping.py b/gguf-py/gguf/tensor_mapping.pyindex ef580518e9..c42a7154a6 100644--- a/gguf-py/gguf/tensor_mapping.py+++ b/gguf-py/gguf/tensor_mapping.py@@ -1355,6 +1355,34 @@ class TensorNameMap:"model.confidence_head.proj", # dspark),+ MODEL_TENSOR.DFLASH_ATTN_CONV_BASE: (+ "model.layers.{bid}.attention_conv.base_kernel",+ ),++ MODEL_TENSOR.DFLASH_ATTN_CONV_PROJ: (+ "model.layers.{bid}.attention_conv.kernel_projection",+ ),++ MODEL_TENSOR.DFLASH_FFN_CONV_BASE: (+ "model.layers.{bid}.mlp_conv.base_kernel",+ ),++ MODEL_TENSOR.DFLASH_FFN_CONV_PROJ: (+ "model.layers.{bid}.mlp_conv.kernel_projection",+ ),++ MODEL_TENSOR.DFLASH_SELECTOR_PREV: (+ "model.candidate_selector.predecessor_codebook",+ ),++ MODEL_TENSOR.DFLASH_SELECTOR_NEXT: (+ "model.candidate_selector.successor_codebook",+ ),++ MODEL_TENSOR.DFLASH_SELECTOR_HIDDEN: (+ "model.candidate_selector.hidden_projection",+ ),+MODEL_TENSOR.CLS: ("classifier", # jina"classifier.dense", # robertadiff --git a/src/llama-arch.cpp b/src/llama-arch.cppindex eecf444fcf..7a9f2f505d 100644--- a/src/llama-arch.cpp+++ b/src/llama-arch.cpp@@ -344,6 +344,12 @@ static const std::map<llm_kv, const char *> LLM_KV_NAMES = {{ LLM_KV_NORM_BEFORE_RESIDUAL, "%s.norm_before_residual" },{ LLM_KV_NORM_BEFORE_FC, "%s.norm_before_fc" },+ { LLM_KV_DFLASH_BLOCK_SIZE, "%s.block_size" },+ { LLM_KV_DFLASH_CONV_KERNEL_SIZE, "%s.conv_kernel_size" },+ { LLM_KV_DFLASH_CONV_GROUP_SIZE, "%s.conv_group_size" },+ { LLM_KV_DFLASH_SELECTOR_RANK, "%s.selector_rank" },+ { LLM_KV_DFLASH_SELECTOR_TOP_K, "%s.selector_top_k" },+{ LLM_KV_SHORTCONV_L_CACHE, "%s.shortconv.l_cache" },// sentence-transformers dense modules feature dims{ LLM_KV_DENSE_2_FEAT_IN, "%s.dense_2_feat_in" },@@ -651,6 +657,13 @@ static const std::map<llm_tensor, const char *> LLM_TENSOR_NAMES = {{ LLM_TENSOR_DSPARK_MARKOV_W1, "markov_w1" },{ LLM_TENSOR_DSPARK_MARKOV_W2, "markov_w2" },{ LLM_TENSOR_DSPARK_CONF_PROJ, "conf_proj" },+ { LLM_TENSOR_DFLASH_ATTN_CONV_BASE, "blk.%d.attn_conv_base" },+ { LLM_TENSOR_DFLASH_ATTN_CONV_PROJ, "blk.%d.attn_conv_proj" },+ { LLM_TENSOR_DFLASH_FFN_CONV_BASE, "blk.%d.ffn_conv_base" },+ { LLM_TENSOR_DFLASH_FFN_CONV_PROJ, "blk.%d.ffn_conv_proj" },+ { LLM_TENSOR_DFLASH_SELECTOR_PREV, "selector_predecessor" },+ { LLM_TENSOR_DFLASH_SELECTOR_NEXT, "selector_successor" },+ { LLM_TENSOR_DFLASH_SELECTOR_HIDDEN, "selector_hidden" },};// declare information about the model weight tensors:@@ -916,6 +929,13 @@ static const std::map<llm_tensor, llm_tensor_info> LLM_TENSOR_INFOS = {{LLM_TENSOR_DSPARK_MARKOV_W1, {LLM_TENSOR_LAYER_OUTPUT, GGML_OP_GET_ROWS}},{LLM_TENSOR_DSPARK_MARKOV_W2, {LLM_TENSOR_LAYER_OUTPUT, GGML_OP_MUL_MAT}},{LLM_TENSOR_DSPARK_CONF_PROJ, {LLM_TENSOR_LAYER_OUTPUT, GGML_OP_MUL_MAT}},+ {LLM_TENSOR_DFLASH_ATTN_CONV_BASE, {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL}},+ {LLM_TENSOR_DFLASH_ATTN_CONV_PROJ, {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL_MAT}},+ {LLM_TENSOR_DFLASH_FFN_CONV_BASE, {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL}},+ {LLM_TENSOR_DFLASH_FFN_CONV_PROJ, {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL_MAT}},+ {LLM_TENSOR_DFLASH_SELECTOR_PREV, {LLM_TENSOR_LAYER_OUTPUT, GGML_OP_GET_ROWS}},+ {LLM_TENSOR_DFLASH_SELECTOR_NEXT, {LLM_TENSOR_LAYER_OUTPUT, GGML_OP_GET_ROWS}},+ {LLM_TENSOR_DFLASH_SELECTOR_HIDDEN, {LLM_TENSOR_LAYER_OUTPUT, GGML_OP_MUL_MAT}},};LLM_KV::LLM_KV(llm_arch arch, const char * suffix) : arch(arch), suffix(suffix) {}diff --git a/src/llama-arch.h b/src/llama-arch.hindex 7159e23bf7..43de4751e0 100644--- a/src/llama-arch.h+++ b/src/llama-arch.h@@ -387,6 +387,11 @@ enum llm_kv {LLM_KV_TARGET_LAYERS,LLM_KV_TARGET_HIDDEN_SIZE,+ LLM_KV_DFLASH_BLOCK_SIZE,+ LLM_KV_DFLASH_CONV_KERNEL_SIZE,+ LLM_KV_DFLASH_CONV_GROUP_SIZE,+ LLM_KV_DFLASH_SELECTOR_RANK,+ LLM_KV_DFLASH_SELECTOR_TOP_K,LLM_KV_NORM_BEFORE_RESIDUAL,LLM_KV_NORM_BEFORE_FC,@@ -659,6 +664,13 @@ enum llm_tensor {LLM_TENSOR_DSPARK_MARKOV_W1,LLM_TENSOR_DSPARK_MARKOV_W2,LLM_TENSOR_DSPARK_CONF_PROJ,+ LLM_TENSOR_DFLASH_ATTN_CONV_BASE,+ LLM_TENSOR_DFLASH_ATTN_CONV_PROJ,+ LLM_TENSOR_DFLASH_FFN_CONV_BASE,+ LLM_TENSOR_DFLASH_FFN_CONV_PROJ,+ LLM_TENSOR_DFLASH_SELECTOR_PREV,+ LLM_TENSOR_DFLASH_SELECTOR_NEXT,+ LLM_TENSOR_DFLASH_SELECTOR_HIDDEN,};diff --git a/src/llama-context.cpp b/src/llama-context.cppindex 0402044da6..22c6522f14 100644--- a/src/llama-context.cpp+++ b/src/llama-context.cpp@@ -2307,6 +2307,10 @@ uint32_t llama_context::graph_max_nodes(uint32_t n_tokens) const {model.arch == LLM_ARCH_MINIMAX_01 ||model.arch == LLM_ARCH_MINIMAX_M3) {res = std::max<uint32_t>(n_tokens * 40, 32u * model.n_tensors());+ } else if (model.arch == LLM_ARCH_DFLASH && model.hparams.dflash_selector_rank > 0) {+ // DFlash2's convolutions and selector are shape work rather than matmuls,+ // so they cost ~8.6 nodes per tensor against ~5.9 for a plain DFlash draft+ res = std::max<uint32_t>(1024u, 12u*model.n_tensors());} else {res = std::max<uint32_t>(1024u, 8u*model.n_tensors());for (const auto & lora : model.loras) {diff --git a/src/llama-ext.h b/src/llama-ext.hindex 35d6e58adf..92a759b7a0 100644--- a/src/llama-ext.h+++ b/src/llama-ext.h@@ -120,6 +120,8 @@ LLAMA_API llama_context * llama_get_ctx_other(struct llama_context * ctx);// model/context data extraction//+LLAMA_API int32_t llama_model_dflash_selector_top_k(const struct llama_model * model);+// returns pointer to the target-model layer indicesLLAMA_API const int32_t * llama_model_target_layer_ids (const struct llama_model * model);// returns the number of extracted layers from target modeldiff --git a/src/llama-hparams.h b/src/llama-hparams.hindex c3c14292c3..1bad6c093b 100644--- a/src/llama-hparams.h+++ b/src/llama-hparams.h@@ -223,6 +223,12 @@ struct llama_hparams {// output embedding dimension (0 = use n_embd)uint32_t n_embd_out_impl = 0;+ uint32_t dflash_block_size = 0;+ uint32_t dflash_conv_kernel_size = 0;+ uint32_t dflash_conv_group_size = 0;+ uint32_t dflash_selector_rank = 0;+ uint32_t dflash_selector_top_k = 0;+// llama4 smallthinkeruint32_t n_moe_layer_step = 0;uint32_t n_no_rope_layer_step = 4;diff --git a/src/llama-model.cpp b/src/llama-model.cppindex ad50093e41..8ab17eb5e9 100644--- a/src/llama-model.cpp+++ b/src/llama-model.cpp@@ -2684,6 +2684,10 @@ int32_t llama_model_n_layer_nextn(const llama_model * model) {return model->hparams.n_layer_nextn;}+int32_t llama_model_dflash_selector_top_k(const llama_model * model) {+ return model->hparams.dflash_selector_top_k;+}+int32_t llama_model_n_head(const llama_model * model) {return model->hparams.n_head();}@@ -2882,6 +2886,10 @@ llama_rope_type llama_model_rope_type(const llama_model * model) {return LLAMA_ROPE_TYPE_NEOX;case LLM_ARCH_DFLASH:+ // drafts for M-RoPE targets carry rope sections and follow the target's temporal dim+ if (const auto & s = model->hparams.rope_sections; s[0] || s[1] || s[2] || s[3]) {+ return LLAMA_ROPE_TYPE_MROPE;+ }// DSV4 DSpark drafters use DeepSeek-V4's normal RoPE; legacy DFlash backbones are NeoXreturn model->hparams.dsv4_hc_mult > 0 ? LLAMA_ROPE_TYPE_NORM : LLAMA_ROPE_TYPE_NEOX;diff --git a/src/llama-model.h b/src/llama-model.hindex 25898ad4af..b85fb23cd9 100644--- a/src/llama-model.h+++ b/src/llama-model.h@@ -363,6 +363,11 @@ struct llama_layer {struct ggml_tensor * ffn_exp_probs_b = nullptr;struct ggml_tensor * ffn_gate_tid2eid = nullptr;+ struct ggml_tensor * dflash_attn_conv_base = nullptr;+ struct ggml_tensor * dflash_attn_conv_proj = nullptr;+ struct ggml_tensor * dflash_ffn_conv_base = nullptr;+ struct ggml_tensor * dflash_ffn_conv_proj = nullptr;+// mamba projstruct ggml_tensor * ssm_in = nullptr;struct ggml_tensor * ssm_x = nullptr;@@ -649,6 +654,10 @@ struct llama_model {struct ggml_tensor * dspark_conf_proj = nullptr;struct ggml_tensor * dspark_conf_proj_b = nullptr;+ struct ggml_tensor * dflash_selector_prev = nullptr;+ struct ggml_tensor * dflash_selector_next = nullptr;+ struct ggml_tensor * dflash_selector_hidden = nullptr;+// unified vector to store target-model extracted layer ids in eagle3, dflash, etc.std::vector<int32_t> target_layer_ids;diff --git a/src/models/dflash.cpp b/src/models/dflash.cppindex ff40c16b22..516651ce40 100644--- a/src/models/dflash.cpp+++ b/src/models/dflash.cpp@@ -7,6 +7,18 @@void llama_model_dflash::load_arch_hparams(llama_model_loader & ml) {ml.get_key(LLM_KV_ATTENTION_LAYERNORM_RMS_EPS, hparams.f_norm_rms_eps);+ ml.get_key(LLM_KV_LOGIT_SCALE, hparams.f_logit_scale, false);+ hparams.f_final_logit_softcapping = 0.0f;+ ml.get_key(LLM_KV_FINAL_LOGIT_SOFTCAPPING, hparams.f_final_logit_softcapping, false);++ // drafts for M-RoPE targets carry degenerate sections [n_rot/2, 0, 0, 0]+ ml.get_key_or_arr(LLM_KV_ROPE_DIMENSION_SECTIONS, hparams.rope_sections, 4, false);++ ml.get_key(LLM_KV_DFLASH_BLOCK_SIZE, hparams.dflash_block_size, false);+ ml.get_key(LLM_KV_DFLASH_CONV_KERNEL_SIZE, hparams.dflash_conv_kernel_size, false);+ ml.get_key(LLM_KV_DFLASH_CONV_GROUP_SIZE, hparams.dflash_conv_group_size, false);+ ml.get_key(LLM_KV_DFLASH_SELECTOR_RANK, hparams.dflash_selector_rank, false);+ ml.get_key(LLM_KV_DFLASH_SELECTOR_TOP_K, hparams.dflash_selector_top_k, false);if (!ml.get_arr(LLM_KV_TARGET_LAYERS, target_layer_ids, false)) {throw std::runtime_error("DFlash model requires 'target_layers' in GGUF metadata");@@ -112,6 +124,29 @@ void llama_model_dflash::load_arch_tensors(llama_model_loader &) {LLAMA_LOG_INFO("%s: DFlash with DSpark markov head (rank = %lld)\n", __func__, (long long) dspark_markov_rank);}+ const struct ggml_tensor * selector_meta = ml->get_tensor_meta("selector_hidden.weight");+ if (selector_meta) {+ const int64_t rank = hparams.dflash_selector_rank;+ if (rank <= 0 || hparams.dflash_block_size <= 0 || hparams.dflash_selector_top_k <= 0 ||+ hparams.dflash_conv_kernel_size <= 0 || hparams.dflash_conv_group_size <= 0) {+ throw std::runtime_error("DFlash2 model is missing conv/selector metadata");+ }+ if (n_embd % hparams.dflash_conv_group_size != 0) {+ throw std::runtime_error("DFlash2 hidden size must be divisible by conv_group_size");+ }+ if (n_embd < hparams.dflash_selector_top_k * (hparams.dflash_selector_top_k + 1)) {+ throw std::runtime_error("DFlash2 hidden size is too small for the selector lattice");+ }++ dflash_selector_prev = create_tensor(tn(LLM_TENSOR_DFLASH_SELECTOR_PREV, "weight"), { rank, n_vocab }, 0);+ dflash_selector_next = create_tensor(tn(LLM_TENSOR_DFLASH_SELECTOR_NEXT, "weight"), { rank, n_vocab }, 0);+ dflash_selector_hidden = create_tensor(tn(LLM_TENSOR_DFLASH_SELECTOR_HIDDEN, "weight"), { n_embd, rank }, 0);++ LLAMA_LOG_INFO("%s: DFlash2 conv kernel = %u, group = %u, selector rank = %u, top-k = %u\n", __func__,+ hparams.dflash_conv_kernel_size, hparams.dflash_conv_group_size,+ hparams.dflash_selector_rank, hparams.dflash_selector_top_k);+ }+fc = create_tensor(tn(LLM_TENSOR_FC, "weight"), { n_embd_inp, n_embd }, 0);fc_s = create_tensor(tn(LLM_TENSOR_FC, "scale"), { 1 }, TENSOR_NOT_REQUIRED);output_norm_enc = create_tensor(tn(LLM_TENSOR_ENC_OUTPUT_NORM, "weight"), { n_embd }, 0); // encoder hidden_norm (after fc)@@ -188,6 +223,16 @@ void llama_model_dflash::load_arch_tensors(llama_model_loader &) {layer.ffn_gate = create_tensor(tn(LLM_TENSOR_FFN_GATE, "weight", i), { n_embd, n_ff }, 0);layer.ffn_down = create_tensor(tn(LLM_TENSOR_FFN_DOWN, "weight", i), { n_ff, n_embd }, 0);layer.ffn_up = create_tensor(tn(LLM_TENSOR_FFN_UP, "weight", i), { n_embd, n_ff }, 0);++ if (selector_meta) {+ const int64_t kernel = hparams.dflash_conv_kernel_size;+ const int64_t groups = n_embd / hparams.dflash_conv_group_size;+ const int64_t projected = 2 * kernel * groups;+ layer.dflash_attn_conv_base = create_tensor(tn(LLM_TENSOR_DFLASH_ATTN_CONV_BASE, i), { n_embd, kernel, 2 }, 0);+ layer.dflash_attn_conv_proj = create_tensor(tn(LLM_TENSOR_DFLASH_ATTN_CONV_PROJ, "weight", i), { n_embd, projected }, 0);+ layer.dflash_ffn_conv_base = create_tensor(tn(LLM_TENSOR_DFLASH_FFN_CONV_BASE, i), { n_embd, kernel, 2 }, 0);+ layer.dflash_ffn_conv_proj = create_tensor(tn(LLM_TENSOR_DFLASH_FFN_CONV_PROJ, "weight", i), { n_embd, projected }, 0);+ }}}@@ -346,6 +391,167 @@ static void build_dspark_markov_head(llm_graph_context & g, const llama_model &ggml_build_forward_expand(g.gf, out);}+static ggml_tensor * build_dflash2_conv(+ llm_graph_context & g,+ ggml_tensor * hidden,+ ggml_tensor * dynamic,+ ggml_tensor * base,+ int side) {+ const auto & hparams = g.hparams;+ const int64_t hidden_size = hidden->ne[0];+ const int64_t n_tokens = hidden->ne[1];+ const int64_t n_blocks = g.ubatch.n_seqs_unq;+ const int64_t kernel_size = hparams.dflash_conv_kernel_size;+ const int64_t group_size = hparams.dflash_conv_group_size;+ const int64_t n_groups = hidden_size / group_size;++ GGML_ASSERT(n_blocks > 0 && n_tokens % n_blocks == 0);+ GGML_ASSERT(dynamic && base && side >= 0 && side < 2);++ const int64_t block_size = n_tokens / n_blocks;+ ggml_context * ctx0 = g.ctx0;+ // ggml_cont copies even when the tensor is already contiguous+ if (!ggml_is_contiguous(hidden) || hidden->ne[1] != n_tokens) {+ hidden = ggml_cont_2d(ctx0, hidden, hidden_size, n_tokens);+ }+ if (!ggml_is_contiguous(dynamic) || dynamic->ne[1] != n_tokens) {+ dynamic = ggml_cont_2d(ctx0, dynamic, dynamic->ne[0], n_tokens);+ }+ ggml_tensor * blocks = ggml_reshape_3d(ctx0, hidden, hidden_size, block_size, n_blocks);+ ggml_tensor * coeffs = ggml_reshape_4d(ctx0, dynamic, n_groups, kernel_size, 2, n_tokens);+ ggml_tensor * coeffs_side = ggml_view_3d(ctx0, coeffs, n_groups, kernel_size, n_tokens,+ coeffs->nb[1], coeffs->nb[3], side * coeffs->nb[2]);++ ggml_tensor * coeff_all = ggml_cont(ctx0, coeffs_side);+ coeff_all = ggml_reshape_4d(ctx0, coeff_all, 1, n_groups, kernel_size, n_tokens);+ coeff_all = ggml_repeat_4d(ctx0, coeff_all, group_size, n_groups, kernel_size, n_tokens);++ ggml_tensor * base_side = ggml_reshape_4d(ctx0,+ ggml_view_1d(ctx0, base, hidden_size * kernel_size, side * base->nb[2]),+ group_size, n_groups, kernel_size, 1);++ ggml_tensor * weight_all = ggml_add(ctx0, coeff_all, base_side);++ ggml_tensor * result = nullptr;+ for (int64_t tap = 0; tap < kernel_size; ++tap) {+ ggml_tensor * values = blocks;+ if (tap > 0) {+ ggml_tensor * zeros = ggml_fill(ctx0,+ ggml_new_tensor_3d(ctx0, hidden->type, hidden_size, std::min(tap, block_size), n_blocks), 0.0f);+ if (tap < block_size) {+ ggml_tensor * previous = ggml_view_3d(ctx0, blocks, hidden_size, block_size - tap, n_blocks,+ blocks->nb[1], blocks->nb[2], 0);+ values = ggml_concat(ctx0, zeros, previous, 1);+ } else {+ values = zeros;+ }+ }+ values = ggml_reshape_2d(ctx0, values, hidden_size, n_tokens);++ ggml_tensor * weight = ggml_reshape_2d(ctx0,+ ggml_cont(ctx0, ggml_view_4d(ctx0, weight_all, group_size, n_groups, 1, n_tokens,+ weight_all->nb[1], weight_all->nb[2], weight_all->nb[3], tap * weight_all->nb[2])),+ hidden_size, n_tokens);++ ggml_tensor * term = ggml_mul(ctx0, weight, values);+ result = result ? ggml_add(ctx0, result, term) : term;+ }+ return result;+}++// DFlash2 selector: top-k candidates per block position plus the pairwise+// transition scores, packed into the nextn output slot for the CPU-side walk.+static void build_dflash2_selector(llm_graph_context & g, const llama_model & model, ggml_tensor * tokens) {+ ggml_context * ctx0 = g.ctx0;+ auto & res = g.res;++ const auto & hparams = g.hparams;+ const int64_t n_tokens = g.n_tokens;+ const int64_t n_embd = g.n_embd;++ const int64_t top_k = hparams.dflash_selector_top_k;+ const int64_t rank = hparams.dflash_selector_rank;+ const int64_t n_blocks = g.ubatch.n_seqs_unq;+ GGML_ASSERT(n_blocks > 0 && n_tokens % n_blocks == 0);+ GGML_ASSERT(res->t_logits->ne[1] == n_tokens);+ if (!tokens) {+ return;+ }++ const int64_t tokens_per_block = n_tokens / n_blocks;+ const int64_t block_size = std::min<int64_t>(tokens_per_block, hparams.dflash_block_size);+ const int64_t row_used = top_k + top_k * top_k;++ ggml_tensor * candidates = ggml_top_k(ctx0, res->t_logits, top_k);+ ggml_tensor * logits_rows = ggml_reshape_3d(ctx0, res->t_logits, 1, res->t_logits->ne[0], n_tokens);+ ggml_tensor * unary = ggml_reshape_2d(ctx0,+ ggml_get_rows(ctx0, logits_rows, candidates), top_k, n_tokens);+ ggml_tensor * gate = g.build_lora_mm(model.dflash_selector_hidden, res->t_embd);++ // Everything below indexes [.., tokens_per_block, n_blocks]: the block+ // position varies fastest, sequences are the outer dimension.+ ggml_tensor * cand_blk = ggml_reshape_3d(ctx0, candidates, top_k, tokens_per_block, n_blocks);+ ggml_tensor * unary_blk = ggml_reshape_3d(ctx0, unary, top_k, tokens_per_block, n_blocks);+ ggml_tensor * gate_blk = ggml_reshape_3d(ctx0, gate, rank, tokens_per_block, n_blocks);++ // a position's score reads only the candidate sets at pos-1 and pos, so a run+ // of positions has no internal dependency and scores in one batched matmul+ auto score_run = [&](int64_t beg_pos, int64_t n_pos, ggml_tensor * pred_ids) {+ ggml_tensor * cand_run = ggml_cont(ctx0, ggml_view_3d(ctx0, cand_blk, top_k, n_pos, n_blocks,+ cand_blk->nb[1], cand_blk->nb[2], beg_pos * cand_blk->nb[1]));+ ggml_tensor * unary_run = ggml_cont(ctx0, ggml_view_3d(ctx0, unary_blk, top_k, n_pos, n_blocks,+ unary_blk->nb[1], unary_blk->nb[2], beg_pos * unary_blk->nb[1]));+ ggml_tensor * gate_run = ggml_cont(ctx0, ggml_view_3d(ctx0, gate_blk, rank, n_pos, n_blocks,+ gate_blk->nb[1], gate_blk->nb[2], beg_pos * gate_blk->nb[1]));++ const int64_t n_pred = pred_ids->ne[0] / (n_pos * n_blocks);++ ggml_tensor * successor = ggml_reshape_4d(ctx0,+ ggml_get_rows(ctx0, model.dflash_selector_next, ggml_reshape_1d(ctx0, cand_run, top_k * n_pos * n_blocks)),+ rank, top_k, n_pos, n_blocks);+ ggml_tensor * predecessor = ggml_reshape_4d(ctx0,+ ggml_get_rows(ctx0, model.dflash_selector_prev, pred_ids),+ rank, n_pred, n_pos, n_blocks);++ ggml_tensor * gate_bcast = ggml_reshape_4d(ctx0, gate_run, rank, 1, n_pos, n_blocks);+ ggml_tensor * cond = ggml_mul(ctx0, predecessor, ggml_repeat(ctx0, gate_bcast, predecessor));+ ggml_tensor * score = ggml_mul_mat(ctx0, successor, cond);+ if (n_pred == 1) {+ score = ggml_repeat_4d(ctx0, score, top_k, top_k, n_pos, n_blocks);+ }+ ggml_tensor * unary_bcast = ggml_reshape_4d(ctx0, unary_run, top_k, 1, n_pos, n_blocks);+ score = ggml_add(ctx0, score, ggml_repeat(ctx0, unary_bcast, score));++ ggml_tensor * row = ggml_concat(ctx0,+ ggml_cast(ctx0, cand_run, GGML_TYPE_F32),+ ggml_reshape_3d(ctx0, score, top_k * top_k, n_pos, n_blocks), 0);+ return ggml_pad(ctx0, row, n_embd - row_used, 0, 0, 0);+ };++ ggml_tensor * packed = ggml_fill(ctx0,+ ggml_new_tensor_3d(ctx0, GGML_TYPE_F32, n_embd, 1, n_blocks), 0.0f);++ if (block_size > 1) {+ // Position 1 alone: its predecessor is the anchor token, one id per+ // sequence rather than a candidate set.+ ggml_tensor * anchor_ids = ggml_cont_1d(ctx0,+ ggml_view_2d(ctx0, tokens, 1, n_blocks, tokens_per_block * tokens->nb[0], 0), n_blocks);+ packed = ggml_concat(ctx0, packed, score_run(1, 1, anchor_ids), 1);+ }+ if (block_size > 2) {+ ggml_tensor * prev_ids = ggml_reshape_1d(ctx0,+ ggml_cont(ctx0, ggml_view_3d(ctx0, cand_blk, top_k, block_size - 2, n_blocks,+ cand_blk->nb[1], cand_blk->nb[2], cand_blk->nb[1])),+ top_k * (block_size - 2) * n_blocks);+ packed = ggml_concat(ctx0, packed, score_run(2, block_size - 2, prev_ids), 1);+ }++ packed = ggml_reshape_2d(ctx0, packed, n_embd, block_size * n_blocks);+ g.cb(packed, "dflash2_lattice", -1);+ res->t_h_nextn = packed;+ ggml_build_forward_expand(g.gf, packed);+}+// DFlash decoder, dual-mode by batch type:// * embd batch -> fused target features: project + inject K/V into the cache.// * token batch -> noise-block diffusion: attend over [committed, MASK...] to generate draft tokens@@ -370,6 +576,20 @@ llama_model_dflash::graph<false>::graph(const llama_model & model, const llm_graconst float kq_scale = 1.0f/sqrtf(float(n_embd_head));+ // drafts for M-RoPE targets use degenerate sections (temporal dim only)+ int sections[4];+ std::copy(std::begin(hparams.rope_sections), std::begin(hparams.rope_sections) + 4, sections);++ auto build_rope = [&](ggml_tensor * cur, ggml_tensor * pos) {+ return rope_type == GGML_ROPE_TYPE_MROPE+ ? ggml_rope_multi(ctx0, cur, pos, nullptr,+ n_rot, sections, rope_type, n_ctx_orig, freq_base, freq_scale,+ ext_factor, attn_factor, beta_fast, beta_slow)+ : ggml_rope_ext(ctx0, cur, pos, nullptr,+ n_rot, rope_type, n_ctx_orig, freq_base, freq_scale,+ ext_factor, attn_factor, beta_fast, beta_slow);+ };+// KV cache injectionif (ubatch.embd) {auto inp = std::make_unique<llm_graph_input_embd>(n_embd);@@ -392,11 +612,7 @@ llama_model_dflash::graph<false>::graph(const llama_model & model, const llm_graVcur = ggml_reshape_3d(ctx0, Vcur, n_embd_head, n_head_kv, n_tokens);Kcur = build_norm(Kcur, layer.attn_k_norm, NULL, LLM_NORM_RMS, il);- Kcur = ggml_rope_ext(- ctx0, Kcur, inp_pos, nullptr,- n_rot, rope_type, n_ctx_orig, freq_base, freq_scale,- ext_factor, attn_factor, beta_fast, beta_slow- );+ Kcur = build_rope(Kcur, inp_pos);cb(Kcur, "Kcur_injected", il);cb(Vcur, "Vcur_injected", il);@@ -450,6 +666,7 @@ llama_model_dflash::graph<false>::graph(const llama_model & model, const llm_grainp->tokens = ggml_new_tensor_1d(ctx0, GGML_TYPE_I32, n_tokens);ggml_set_input(inp->tokens);+ res->t_inp_tokens = inp->tokens;ggml_tensor * inp_tokens = inp->tokens;@@ -464,6 +681,13 @@ llama_model_dflash::graph<false>::graph(const llama_model & model, const llm_graggml_tensor * noise_norm = build_norm(inpL, layer.attn_norm, NULL, LLM_NORM_RMS, il);cb(noise_norm, "noise_norm", il);+ ggml_tensor * attn_dynamic = nullptr;+ if (layer.dflash_attn_conv_proj) {+ attn_dynamic = build_lora_mm(layer.dflash_attn_conv_proj, noise_norm);+ noise_norm = build_dflash2_conv(*this, noise_norm, attn_dynamic, layer.dflash_attn_conv_base, 0);+ cb(noise_norm, "attn_conv_in", il);+ }+ggml_tensor * Qcur = build_lora_mm(layer.wq, noise_norm);ggml_tensor * Kcur = build_lora_mm(layer.wk, noise_norm);ggml_tensor * Vcur = build_lora_mm(layer.wv, noise_norm);@@ -475,16 +699,8 @@ llama_model_dflash::graph<false>::graph(const llama_model & model, const llm_graQcur = build_norm(Qcur, layer.attn_q_norm, NULL, LLM_NORM_RMS, il);Kcur = build_norm(Kcur, layer.attn_k_norm, NULL, LLM_NORM_RMS, il);- Qcur = ggml_rope_ext(- ctx0, Qcur, inp_pos, nullptr,- n_rot, rope_type, n_ctx_orig, freq_base, freq_scale,- ext_factor, attn_factor, beta_fast, beta_slow- );- Kcur = ggml_rope_ext(- ctx0, Kcur, inp_pos, nullptr,- n_rot, rope_type, n_ctx_orig, freq_base, freq_scale,- ext_factor, attn_factor, beta_fast, beta_slow- );+ Qcur = build_rope(Qcur, inp_pos);+ Kcur = build_rope(Kcur, inp_pos);cb(Qcur, "Qcur", il);cb(Kcur, "Kcur", il);cb(Vcur, "Vcur", il);@@ -494,12 +710,24 @@ llama_model_dflash::graph<false>::graph(const llama_model & model, const llm_gra? build_attn(inp_attn_iswa, layer.wo, NULL, NULL, Qcur, Kcur, Vcur, nullptr, nullptr, nullptr, kq_scale, il): build_attn(inp_attn, layer.wo, NULL, NULL, Qcur, Kcur, Vcur, nullptr, nullptr, nullptr, kq_scale, il);+ if (attn_dynamic) {+ cur = build_dflash2_conv(*this, cur, attn_dynamic, layer.dflash_attn_conv_base, 1);+ cb(cur, "attn_conv_out", il);+ }+ggml_tensor * ffn_inp = ggml_add(ctx0, cur, inpL);cb(ffn_inp, "ffn_inp", il);cur = build_norm(ffn_inp, layer.ffn_norm, NULL, LLM_NORM_RMS, il);cb(cur, "ffn_norm", il);+ ggml_tensor * ffn_dynamic = nullptr;+ if (layer.dflash_ffn_conv_proj) {+ ffn_dynamic = build_lora_mm(layer.dflash_ffn_conv_proj, cur);+ cur = build_dflash2_conv(*this, cur, ffn_dynamic, layer.dflash_ffn_conv_base, 0);+ cb(cur, "ffn_conv_in", il);+ }+cur = build_ffn(cur,layer.ffn_up, NULL, layer.ffn_up_s,layer.ffn_gate, NULL, layer.ffn_gate_s,@@ -508,6 +736,11 @@ llama_model_dflash::graph<false>::graph(const llama_model & model, const llm_graLLM_FFN_SILU, LLM_FFN_PAR, il);cb(cur, "ffn_out", il);+ if (ffn_dynamic) {+ cur = build_dflash2_conv(*this, cur, ffn_dynamic, layer.dflash_ffn_conv_base, 1);+ cb(cur, "ffn_conv_out", il);+ }+cur = ggml_add(ctx0, cur, ffn_inp);cb(cur, "l_out", il);@@ -532,6 +765,19 @@ llama_model_dflash::graph<false>::graph(const llama_model & model, const llm_gracur = build_lora_mm(output, cur, output_s);+ // DFlash2 feeds these logits to the selector, so they need the target's output+ // transforms; DFlash1 and DSpark read them through the sampler instead+ if (model.dflash_selector_hidden) {+ if (hparams.f_logit_scale != 0.0f) {+ cur = ggml_scale(ctx0, cur, hparams.f_logit_scale);+ }+ if (hparams.f_final_logit_softcapping > 0.0f) {+ cur = ggml_scale(ctx0, cur, 1.0f / hparams.f_final_logit_softcapping);+ cur = ggml_tanh(ctx0, cur);+ cur = ggml_scale(ctx0, cur, hparams.f_final_logit_softcapping);+ }+ }+// reduced-draft-vocab exports: scatter the draft logits to the target vocabulary via d2tif (model.d2t) {const int64_t n_draft_vocab = cur->ne[0];@@ -556,6 +802,10 @@ llama_model_dflash::graph<false>::graph(const llama_model & model, const llm_graif (model.dspark_markov_w1) {build_dspark_markov_head(*this, model, inp_tokens);}++ if (model.dflash_selector_hidden) {+ build_dflash2_selector(*this, model, inp_tokens);+ }}// DSV4 DSpark decoder, dual-mode by batch type (see the DFlash decoder above):diff --git a/tests/test-backend-ops.cpp b/tests/test-backend-ops.cppindex b1d86a4e21..df0b9aa666 100644--- a/tests/test-backend-ops.cpp+++ b/tests/test-backend-ops.cpp@@ -9717,6 +9717,17 @@ static std::vector<std::unique_ptr<test_case>> make_test_cases_eval() {}}}+ for (int k : {4, 8, 16, 32}) {+ for (int nrows : {1, 8, 16}) {+ test_cases.emplace_back(new test_top_k(GGML_TYPE_F32, {202048, nrows, 1, 1}, k));+ test_cases.emplace_back(new test_top_k(GGML_TYPE_F32, {151936, nrows, 1, 1}, k));+ test_cases.emplace_back(new test_top_k(GGML_TYPE_F32, {8192, nrows, 1, 1}, k));+ test_cases.emplace_back(new test_top_k(GGML_TYPE_F32, {8193, nrows, 1, 1}, k));+ test_cases.emplace_back(new test_top_k(GGML_TYPE_F32, {8192, nrows, 1, 1}, k, true));+ test_cases.emplace_back(new test_top_k(GGML_TYPE_F32, {202048, nrows, 1, 1}, k, true));+ }+ }+for (int k : {1, 2, 3, 7, 15}) {test_cases.emplace_back(new test_top_k(GGML_TYPE_F32, {16, 10, 10, 10}, k));test_cases.emplace_back(new test_top_k(GGML_TYPE_F32, {60, 10, 10, 10}, k));@@ -10454,7 +10465,13 @@ static std::vector<std::unique_ptr<test_case>> make_test_cases_perf() {test_cases.emplace_back(new test_argsort(GGML_TYPE_F32, {200000, 16, 1, 1}));test_cases.emplace_back(new test_top_k(GGML_TYPE_F32, {2, 1, 1, 1}, 1));- for (auto k : {1, 10, 40, 400}) {+ // widths around the tiling threshold+ for (auto cols : {4096, 8192, 12288, 16384, 24576, 32768, 65536, 131072}) {+ for (auto nrows : {1, 16}) {+ test_cases.emplace_back(new test_top_k(GGML_TYPE_F32, {cols, nrows, 1, 1}, 16));+ }+ }+ for (auto k : {1, 4, 8, 10, 16, 32, 40, 400}) {for (auto nrows : {1, 16}) {for (auto cols : {k, 1000, 65000, 200000}) {test_cases.emplace_back(new test_top_k(GGML_TYPE_F32, {cols, nrows, 1, 1}, k));