LLM inference in C/C++
C++ ggml
Dflash support for nemotron-3.5 (#26905)
lnigamcc078b45b635
diff --git a/conversion/base.py b/conversion/base.pyindex 3572b77c21..718d539449 100644--- a/conversion/base.py+++ b/conversion/base.py@@ -829,7 +829,7 @@ class ModelBase:elif any(str(v.get("quant_algo")).endswith("NVFP4") for v in quant_layers.values() if isinstance(v, dict)):quant_algo = "NVFP4"- self._is_nvfp4 = quant_algo == "NVFP4"+ self._is_nvfp4 = quant_algo in ("NVFP4", "W4A16_NVFP4")self._is_mxfp4 = quant_method == "mxfp4"# NVFP4 weights are repacked and written directly to gguf_writer.diff --git a/conversion/qwen.py b/conversion/qwen.pyindex b4ae528bf2..ead435455d 100644--- a/conversion/qwen.py+++ b/conversion/qwen.py@@ -647,10 +647,13 @@ class DFlashModel(Qwen3Model):# own tokenizer logic, not the Qwen default).from . import get_model_classwith open(self.target_model_dir / "config.json", "r", encoding="utf-8") as f:- target_arch = json.load(f)["architectures"][0]+ target_hparams = json.load(f)+ target_arch = target_hparams["architectures"][0]target_cls = get_model_class(target_arch)if target_cls is not type(self):+ if target_arch == "NemotronHForCausalLM":+ setattr(self, "is_moe", "num_experts_per_tok" in target_hparams)target_cls.set_vocab(self) # ty: ignore[unresolved-attribute]else:super().set_vocab()@@ -688,6 +691,12 @@ class DFlashModel(Qwen3Model):name = "model." + namereturn super().filter_tensors((name, gen))+ def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]:+ if name == "model.embed_tokens.weight" and not self.hparams.get("has_embed_tokens", True):+ return++ yield from super().modify_tensors(data_torch, name, bid)+@ModelBase.register("Qwen3DSparkModel")class DSparkModel(DFlashModel):diff --git a/gguf-py/gguf/constants.py b/gguf-py/gguf/constants.pyindex a197cacd3f..98c4fa1691 100644--- a/gguf-py/gguf/constants.py+++ b/gguf-py/gguf/constants.py@@ -4726,6 +4726,7 @@ MODEL_TENSORS: dict[MODEL_ARCH, list[MODEL_TENSOR]] = {MODEL_TENSOR.D2T,],MODEL_ARCH.DFLASH: [+ MODEL_TENSOR.TOKEN_EMBD,MODEL_TENSOR.OUTPUT_NORM,MODEL_TENSOR.ATTN_NORM,MODEL_TENSOR.ATTN_Q,diff --git a/src/llama-model.h b/src/llama-model.hindex 1dd0904387..341cb66fba 100644--- a/src/llama-model.h+++ b/src/llama-model.h@@ -623,8 +623,9 @@ struct llama_model {struct ggml_tensor * per_layer_model_proj = nullptr;struct ggml_tensor * per_layer_proj_norm = nullptr;- // eagle3- struct ggml_tensor * fc = nullptr; // feature fusion layer+ // eagle3 / dflash feature fusion layer+ struct ggml_tensor * fc = nullptr;+ struct ggml_tensor * fc_s = nullptr;struct ggml_tensor * d2t = nullptr; // draft to target vocabulary mapping// dsparkdiff --git a/src/models/dflash.cpp b/src/models/dflash.cppindex daff6e78f1..bfbdb28ee5 100644--- a/src/models/dflash.cpp+++ b/src/models/dflash.cpp@@ -79,6 +79,7 @@ void llama_model_dflash::load_arch_tensors(llama_model_loader &) {const int64_t n_embd_inp = hparams.n_embd_inp_enc();+ tok_embd = create_tensor(tn(LLM_TENSOR_TOKEN_EMBD, "weight"), { n_embd, n_vocab }, TENSOR_NOT_REQUIRED);// DSpark = DFlash + a semi-autoregressive Markov head and Confidence head//// TODO: only Qwen3-style backbones are supported for now; other backbones (e.g. Gemma4)@@ -97,6 +98,7 @@ void llama_model_dflash::load_arch_tensors(llama_model_loader &) {}fc = create_tensor(tn(LLM_TENSOR_FC, "weight"), { n_embd_inp, n_embd }, 0);+ fc_s = create_tensor(tn(LLM_TENSOR_FC, "scale"), { 1 }, TENSOR_NOT_REQUIRED);output_norm_enc = create_tensor(tn(LLM_TENSOR_ENC_OUTPUT_NORM, "weight"), { n_embd }, 0); // encoder hidden_norm (after fc)output_norm = create_tensor(tn(LLM_TENSOR_OUTPUT_NORM, "weight"), { n_embd }, 0); // decoder final norm@@ -205,7 +207,7 @@ template <>llama_model_dflash::graph<true>::graph(const llama_model & model, const llm_graph_params & params) : llm_graph_context(params) {ggml_tensor * cur = build_inp_embd_enc();- cur = build_lora_mm(model.fc, cur);+ cur = build_lora_mm(model.fc, cur, model.fc_s);cb(cur, "fc_out", -1);cur = build_norm(cur, model.output_norm_enc, NULL, LLM_NORM_RMS, -1);@@ -460,9 +462,9 @@ llama_model_dflash::graph<false>::graph(const llama_model & model, const llm_gracb(cur, "ffn_norm", il);cur = build_ffn(cur,- layer.ffn_up, NULL, NULL,- layer.ffn_gate, NULL, NULL,- layer.ffn_down, NULL, NULL,+ layer.ffn_up, NULL, layer.ffn_up_s,+ layer.ffn_gate, NULL, layer.ffn_gate_s,+ layer.ffn_down, NULL, layer.ffn_down_s,NULL,LLM_FFN_SILU, LLM_FFN_PAR, il);cb(cur, "ffn_out", il);@@ -479,15 +481,17 @@ llama_model_dflash::graph<false>::graph(const llama_model & model, const llm_grares->t_embd = cur;// lm_head from the target model (shared via ctx_other)- auto * output = model.output;+ auto * output = model.output;+ auto * output_s = model.output_s;if (output == nullptr) {GGML_ASSERT(cparams.ctx_other != nullptr);const auto * model_other = llama_get_model(cparams.ctx_other);GGML_ASSERT(model_other->output != nullptr && "DFlash decoder requires the target model's output projection");- output = model_other->output;+ output = model_other->output;+ output_s = model_other->output_s;}- cur = build_lora_mm(output, cur);+ cur = build_lora_mm(output, cur, output_s);cb(cur, "result_output", -1);res->t_logits = cur;@@ -655,15 +659,17 @@ llama_model_dflash::graph_dsv4::graph_dsv4(const llama_model & model, const llm_cb(cur, "result_norm", -1);// lm_head from the target model (shared via ctx_other)- auto * output = model.output;+ auto * output = model.output;+ auto * output_s = model.output_s;if (output == nullptr) {GGML_ASSERT(cparams.ctx_other != nullptr);const auto * model_other = llama_get_model(cparams.ctx_other);GGML_ASSERT(model_other->output != nullptr && "DSpark decoder requires the target model's output projection");- output = model_other->output;+ output = model_other->output;+ output_s = model_other->output_s;}- cur = build_lora_mm(output, cur);+ cur = build_lora_mm(output, cur, output_s);cb(cur, "result_output", -1);res->t_logits = cur;diff --git a/src/models/nemotron-h.cpp b/src/models/nemotron-h.cppindex cd2af3179c..f02674c646 100644--- a/src/models/nemotron-h.cpp+++ b/src/models/nemotron-h.cpp@@ -177,8 +177,11 @@ llama_model_nemotron_h::graph::graph(const llama_model & model, const llm_graph_auto * inp = build_inp_mem_hybrid();ggml_tensor * inp_out_ids = build_inp_out_ids();+ const bool extract_final_inp = (size_t) n_layer < cparams.embeddings_layer_inp.size() && cparams.embeddings_layer_inp[n_layer];for (int il = 0; il < n_layer; ++il) {+ res->t_layer_inp[il] = inpL;+struct ggml_tensor * inpSA = inpL;// norm@@ -195,7 +198,7 @@ llama_model_nemotron_h::graph::graph(const llama_model & model, const llm_graph_cur = build_ffn_layer(cur, model, il);}- if (il == n_layer - 1 && inp_out_ids && cparams.embeddings_nextn_masked) {+ if (il == n_layer - 1 && inp_out_ids && cparams.embeddings_nextn_masked && !extract_final_inp) {cur = ggml_get_rows(ctx0, cur, inp_out_ids);inpSA = ggml_get_rows(ctx0, inpSA, inp_out_ids);}@@ -209,6 +212,13 @@ llama_model_nemotron_h::graph::graph(const llama_model & model, const llm_graph_}cur = inpL;+ if (extract_final_inp) {+ res->t_layer_inp[n_layer] = cur;++ if (inp_out_ids && cparams.embeddings_nextn_masked) {+ cur = ggml_get_rows(ctx0, cur, inp_out_ids);+ }+ }cur = build_norm(cur, model.output_norm, NULL, LLM_NORM_RMS, -1);