LLM inference in C/C++
C++ ggml
model: correctly support input vision for deepseek4 (#28154)
Xuan-Son Nguyen9400c8946e4d
diff --git a/conversion/deepseek.py b/conversion/deepseek.pyindex c244e94ec3..abaed7880b 100644--- a/conversion/deepseek.py+++ b/conversion/deepseek.py@@ -578,8 +578,7 @@ class DeepseekV4Model(TextModel):@classmethoddef filter_tensors(cls, item: tuple[str, Callable[[], Tensor]]) -> tuple[str, Callable[[], Tensor]] | None:name, gen = item- if (name.startswith(("aligner.", "image_"))- or name.endswith(".ffn.gate.bias_vl")):+ if name.startswith(("aligner.", "image_")):return Noneif name.startswith("mtp."):if not cls.mtp_only:@@ -856,6 +855,7 @@ class DeepseekV4Model(TextModel):"ffn_norm.weight": (gguf.MODEL_TENSOR.FFN_NORM, ".weight"),"ffn.gate.weight": (gguf.MODEL_TENSOR.FFN_GATE_INP, ".weight"),"ffn.gate.bias": (gguf.MODEL_TENSOR.FFN_EXP_PROBS_B, ".bias"),+ "ffn.gate.bias_vl": (gguf.MODEL_TENSOR.FFN_EXP_PROBS_B_VL, ".bias"),"ffn.gate.tid2eid": (gguf.MODEL_TENSOR.FFN_GATE_TID2EID, ".weight"),"ffn.shared_experts.w1.weight": (gguf.MODEL_TENSOR.FFN_GATE_SHEXP, ".weight"),"ffn.shared_experts.w2.weight": (gguf.MODEL_TENSOR.FFN_DOWN_SHEXP, ".weight"),@@ -881,6 +881,10 @@ class DeepseekV4Model(TextModel):if re.match(r"layers\.\d+\.ffn\.experts\.\d+\.w[123]\.(weight|scale)$", name):return []+ # hash layers route text tokens via tid2eid and image tokens via bias_vl; gate.bias is unused+ if name.endswith(".ffn.gate.bias") and bid is not None and bid < self.hparams["num_hash_layers"]:+ return []+tensor_key, suffix = self._map_dsv4_tensor_name(name, bid)if tensor_key == gguf.MODEL_TENSOR.FFN_GATE_TID2EID:return []diff --git a/gguf-py/gguf/constants.py b/gguf-py/gguf/constants.pyindex 56477c1988..b85f62a311 100644--- a/gguf-py/gguf/constants.py+++ b/gguf-py/gguf/constants.py@@ -697,6 +697,7 @@ class MODEL_TENSOR(IntEnum):FFN_DOWN_CHEXP = auto()FFN_UP_CHEXP = auto()FFN_EXP_PROBS_B = auto()+ FFN_EXP_PROBS_B_VL = auto() # deepseek4 vision (bias for image tokens)FFN_GATE_TID2EID = auto()MOE_LATENT_DOWN = auto() # nemotron 3 superMOE_LATENT_UP = auto() # nemotron 3 super@@ -1449,6 +1450,7 @@ TENSOR_NAMES: dict[MODEL_TENSOR, str] = {MODEL_TENSOR.FFN_UP_EXP: "blk.{bid}.ffn_up_exps",MODEL_TENSOR.FFN_GATE_UP_EXP: "blk.{bid}.ffn_gate_up_exps",MODEL_TENSOR.FFN_EXP_PROBS_B: "blk.{bid}.exp_probs_b",+ MODEL_TENSOR.FFN_EXP_PROBS_B_VL: "blk.{bid}.exp_probs_b_vl",MODEL_TENSOR.FFN_GATE_TID2EID: "blk.{bid}.ffn_gate_tid2eid",MODEL_TENSOR.MOE_LATENT_DOWN: "blk.{bid}.ffn_latent_down", # nemotron 3 superMODEL_TENSOR.MOE_LATENT_UP: "blk.{bid}.ffn_latent_up", # nemotron 3 super@@ -3839,6 +3841,7 @@ MODEL_TENSORS: dict[MODEL_ARCH, list[MODEL_TENSOR]] = {MODEL_TENSOR.FFN_GATE_INP,MODEL_TENSOR.FFN_GATE_TID2EID,MODEL_TENSOR.FFN_EXP_PROBS_B,+ MODEL_TENSOR.FFN_EXP_PROBS_B_VL,MODEL_TENSOR.FFN_NORM,MODEL_TENSOR.FFN_GATE_EXP,MODEL_TENSOR.FFN_DOWN_EXP,diff --git a/src/llama-arch.cpp b/src/llama-arch.cppindex 7adb87411a..446de4ae25 100644--- a/src/llama-arch.cpp+++ b/src/llama-arch.cpp@@ -457,6 +457,7 @@ static const std::map<llm_tensor, const char *> LLM_TENSOR_NAMES = {{ LLM_TENSOR_FFN_UP_SHEXP, "blk.%d.ffn_up_shexp" },{ LLM_TENSOR_FFN_DOWN_SHEXP, "blk.%d.ffn_down_shexp" },{ LLM_TENSOR_FFN_EXP_PROBS_B, "blk.%d.exp_probs_b" },+ { LLM_TENSOR_FFN_EXP_PROBS_B_VL, "blk.%d.exp_probs_b_vl" },{ LLM_TENSOR_FFN_LATENT_DOWN, "blk.%d.ffn_latent_down" },{ LLM_TENSOR_FFN_LATENT_UP, "blk.%d.ffn_latent_up" },{ LLM_TENSOR_ATTN_NORM_2, "blk.%d.attn_norm_2" },@@ -896,6 +897,7 @@ static const std::map<llm_tensor, llm_tensor_info> LLM_TENSOR_INFOS = {{LLM_TENSOR_FFN_GATE_CHEXPS, {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL_MAT_ID}},{LLM_TENSOR_FFN_UP_CHEXPS, {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL_MAT_ID}},{LLM_TENSOR_FFN_EXP_PROBS_B, {LLM_TENSOR_LAYER_REPEATING, GGML_OP_ADD}},+ {LLM_TENSOR_FFN_EXP_PROBS_B_VL, {LLM_TENSOR_LAYER_REPEATING, GGML_OP_ADD}},// altup / laurel (gemma 3n){LLM_TENSOR_PER_LAYER_TOKEN_EMBD, {LLM_TENSOR_LAYER_INPUT, GGML_OP_GET_ROWS}},{LLM_TENSOR_PER_LAYER_MODEL_PROJ, {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL_MAT}},diff --git a/src/llama-arch.h b/src/llama-arch.hindex ca7d55a5fd..0c0b994836 100644--- a/src/llama-arch.h+++ b/src/llama-arch.h@@ -477,6 +477,7 @@ enum llm_tensor {LLM_TENSOR_FFN_GATE_CHEXPS,LLM_TENSOR_FFN_UP_CHEXPS,LLM_TENSOR_FFN_EXP_PROBS_B,+ LLM_TENSOR_FFN_EXP_PROBS_B_VL,LLM_TENSOR_FFN_LATENT_DOWN,LLM_TENSOR_FFN_LATENT_UP,LLM_TENSOR_ATTN_Q_NORM,diff --git a/src/llama-hparams.h b/src/llama-hparams.hindex 1411692a89..390d3543d7 100644--- a/src/llama-hparams.h+++ b/src/llama-hparams.h@@ -161,6 +161,10 @@ struct llama_hparams {// the size of the sliding window (0 - no SWA)uint32_t n_swa = 0;+ // deepseek4 vision: when decoding non-causally (multimodal input), SWA is not applied between tokens of the current ubatch (the image span); older tokens are still window-clipped+ // for other models (like gemma 3, gemma 4): SWA is always applied to match transformers implementation+ bool swa_full_non_causal = false;+// if is_swa_impl[il] == 1, then layer il is SWA// if is_swa_impl[il] == 0, then layer il is dense (i.e. non-SWA)// by default, all layers are densediff --git a/src/llama-kv-cache.cpp b/src/llama-kv-cache.cppindex 3e4a4d56f9..f22054c3d6 100644--- a/src/llama-kv-cache.cpp+++ b/src/llama-kv-cache.cpp@@ -1681,7 +1681,9 @@ static void set_input_kq_mask_impl(const args_set_input_kq_mask & args, T * data// apply SWA if anyif (swa) {- if (llama_hparams::is_masked_swa(n_swa, swa_type, p0, p1)) {+ // see llama_hparams::swa_full_non_causal+ const bool in_span = !causal && args.hparams.swa_full_non_causal && p0 >= seq_pos_min[seq_id];+ if (!in_span && llama_hparams::is_masked_swa(n_swa, swa_type, p0, p1)) {goto skip;}}diff --git a/src/llama-model.h b/src/llama-model.hindex 38066538ed..ee6bb5ac37 100644--- a/src/llama-model.h+++ b/src/llama-model.h@@ -362,6 +362,7 @@ struct llama_layer {struct ggml_tensor * ffn_up_b = nullptr; // b3struct ggml_tensor * ffn_act = nullptr;struct ggml_tensor * ffn_exp_probs_b = nullptr;+ struct ggml_tensor * ffn_exp_probs_b_vl = nullptr; // deepseek4 vision (bias for image tokens)struct ggml_tensor * ffn_gate_tid2eid = nullptr;struct ggml_tensor * dflash_attn_conv_base = nullptr;diff --git a/src/models/deepseek4.cpp b/src/models/deepseek4.cppindex 222f222492..680516b230 100644--- a/src/models/deepseek4.cpp+++ b/src/models/deepseek4.cpp@@ -66,6 +66,9 @@ void llama_model_deepseek4::load_arch_hparams(llama_model_loader & ml) {}hparams.swa_type = LLAMA_SWA_TYPE_STANDARD;hparams.set_swa_pattern(0);+ // tokens of an image span attend bidirectionally to the whole span, the window only applies to older tokens+ // ref: get_window_topk_idxs_visible in the reference impl+ hparams.swa_full_non_causal = true;for (uint32_t il = hparams.n_layer(); il < hparams.n_layer_all; ++il) {hparams.is_swa_impl[il] = true;}@@ -156,6 +159,8 @@ void llama_model_deepseek4::load_arch_tensors(llama_model_loader & ml) {} else {layer.ffn_exp_probs_b = create_tensor(tn(LLM_TENSOR_FFN_EXP_PROBS_B, "bias", i), {n_expert}, flags);}+ // vision variant only: routing bias for image tokens+ layer.ffn_exp_probs_b_vl = create_tensor(tn(LLM_TENSOR_FFN_EXP_PROBS_B_VL, "bias", i), {n_expert}, flags | TENSOR_NOT_REQUIRED);layer.ffn_norm = create_tensor(tn(LLM_TENSOR_FFN_NORM, "weight", i), {n_embd}, flags);layer.ffn_gate_exps = create_tensor(tn(LLM_TENSOR_FFN_GATE_EXPS, "weight", i), {n_embd, n_ff_exp, n_expert}, flags);@@ -1275,7 +1280,14 @@ llama_model_deepseek4::graph::graph(const llama_model & model, const llm_graph_pconst auto & layer = model.layers[il];ggml_tensor * selected_experts = nullptr;ggml_tensor * exp_probs_b = layer.ffn_exp_probs_b;- if ((uint32_t) il < hparams.dsv4_hash_layer_count) {++ // may apply exp_probs_b_vl is input is from mtmd+ const bool is_media = ubatch.embd != nullptr;+ if (is_media) {+ if (layer.ffn_exp_probs_b_vl) {+ exp_probs_b = layer.ffn_exp_probs_b_vl;+ }+ } else if ((uint32_t) il < hparams.dsv4_hash_layer_count) {selected_experts = ggml_get_rows(ctx0, layer.ffn_gate_tid2eid, res->t_inp_tokens);exp_probs_b = nullptr;}diff --git a/tools/mtmd/mtmd.cpp b/tools/mtmd/mtmd.cppindex d2b88b1e46..f39f0974e3 100644--- a/tools/mtmd/mtmd.cpp+++ b/tools/mtmd/mtmd.cpp@@ -2132,6 +2132,7 @@ bool mtmd_decode_use_non_causal(const mtmd_context * ctx, const mtmd_input_chunkcase PROJECTOR_TYPE_GEMMA3:case PROJECTOR_TYPE_GEMMA4V:case PROJECTOR_TYPE_GEMMA4UV:+ case PROJECTOR_TYPE_DEEPSEEK4V:return true;default:return false;