LLM inference in C/C++
C++ ggml
qwen3 : bidirectional attention and dense_2 output projection (#30607)
fzowl412a363c655b
diff --git a/conversion/base.py b/conversion/base.pyindex 72c1256ba7..14a52f01d4 100644--- a/conversion/base.py+++ b/conversion/base.py@@ -1625,7 +1625,7 @@ class TextModel(ModelBase):self.gguf_writer.add_head_count_kv(n_head_kv)logger.info(f"gguf: key-value head count = {n_head_kv}")- if self.hparams.get("is_causal") is False:+ if self.hparams.get("is_causal") is False or self.hparams.get("use_bidirectional_attention") is True:self.gguf_writer.add_causal_attention(False)logger.info("gguf: causal attention = False")diff --git a/conversion/qwen.py b/conversion/qwen.pyindex 43d2fb466f..0c0d0c1edc 100644--- a/conversion/qwen.py+++ b/conversion/qwen.py@@ -253,6 +253,11 @@ class Qwen3Model(Qwen2Model):yield from super().modify_tensors(data_torch, name, bid)return+ # output embedding projection (voyage-4-nano)+ if name == "linear.weight":+ self.gguf_writer.add_embedding_length_out(data_torch.shape[0])+ name = "dense_2_out.weight"+yield from super().modify_tensors(data_torch, name, bid)diff --git a/gguf-py/gguf/constants.py b/gguf-py/gguf/constants.pyindex 32e088d882..e098efae42 100644--- a/gguf-py/gguf/constants.py+++ b/gguf-py/gguf/constants.py@@ -2929,6 +2929,7 @@ MODEL_TENSORS: dict[MODEL_ARCH, list[MODEL_TENSOR]] = {MODEL_TENSOR.FFN_GATE,MODEL_TENSOR.FFN_DOWN,MODEL_TENSOR.FFN_UP,+ MODEL_TENSOR.DENSE_2_OUT, # voyage-4-nano],MODEL_ARCH.QWEN3MOE: [MODEL_TENSOR.TOKEN_EMBD,diff --git a/src/models/qwen3.cpp b/src/models/qwen3.cppindex bf7e622fb9..c01f838292 100644--- a/src/models/qwen3.cpp+++ b/src/models/qwen3.cpp@@ -28,6 +28,10 @@ void llama_model_qwen3::load_arch_tensors(llama_model_loader &) {// output rerank headcls_out = create_tensor(tn(LLM_TENSOR_CLS_OUT, "weight"), {n_embd, hparams.n_cls_out}, TENSOR_NOT_REQUIRED);+ // output embedding projection+ dense_2_out_layers = create_tensor(tn(LLM_TENSOR_DENSE_2_OUT, "weight"), {n_embd, hparams.n_embd_out()}, TENSOR_NOT_REQUIRED);+ dense_2_out_layers_b = create_tensor(tn(LLM_TENSOR_DENSE_2_OUT, "bias"), {hparams.n_embd_out() }, TENSOR_NOT_REQUIRED);+for (int i = 0; i < n_layer; ++i) {auto & layer = layers[i];