Skip to content

LLM inference in C/C++

C++ ggml

qwen3 : bidirectional attention and dense_2 output projection (#30607)

fzowl412a363c655b
diff --git a/conversion/base.py b/conversion/base.py
index 72c1256ba7..14a52f01d4 100644
--- a/conversion/base.py
+++ b/conversion/base.py
@@ -1625,7 +1625,7 @@ class TextModel(ModelBase):
self.gguf_writer.add_head_count_kv(n_head_kv)
logger.info(f"gguf: key-value head count = {n_head_kv}")
- if self.hparams.get("is_causal") is False:
+ if self.hparams.get("is_causal") is False or self.hparams.get("use_bidirectional_attention") is True:
self.gguf_writer.add_causal_attention(False)
logger.info("gguf: causal attention = False")
diff --git a/conversion/qwen.py b/conversion/qwen.py
index 43d2fb466f..0c0d0c1edc 100644
--- a/conversion/qwen.py
+++ b/conversion/qwen.py
@@ -253,6 +253,11 @@ class Qwen3Model(Qwen2Model):
yield from super().modify_tensors(data_torch, name, bid)
return
+ # output embedding projection (voyage-4-nano)
+ if name == "linear.weight":
+ self.gguf_writer.add_embedding_length_out(data_torch.shape[0])
+ name = "dense_2_out.weight"
+
yield from super().modify_tensors(data_torch, name, bid)
diff --git a/gguf-py/gguf/constants.py b/gguf-py/gguf/constants.py
index 32e088d882..e098efae42 100644
--- a/gguf-py/gguf/constants.py
+++ b/gguf-py/gguf/constants.py
@@ -2929,6 +2929,7 @@ MODEL_TENSORS: dict[MODEL_ARCH, list[MODEL_TENSOR]] = {
MODEL_TENSOR.FFN_GATE,
MODEL_TENSOR.FFN_DOWN,
MODEL_TENSOR.FFN_UP,
+ MODEL_TENSOR.DENSE_2_OUT, # voyage-4-nano
],
MODEL_ARCH.QWEN3MOE: [
MODEL_TENSOR.TOKEN_EMBD,
diff --git a/src/models/qwen3.cpp b/src/models/qwen3.cpp
index bf7e622fb9..c01f838292 100644
--- a/src/models/qwen3.cpp
+++ b/src/models/qwen3.cpp
@@ -28,6 +28,10 @@ void llama_model_qwen3::load_arch_tensors(llama_model_loader &) {
// output rerank head
cls_out = create_tensor(tn(LLM_TENSOR_CLS_OUT, "weight"), {n_embd, hparams.n_cls_out}, TENSOR_NOT_REQUIRED);
+ // output embedding projection
+ dense_2_out_layers = create_tensor(tn(LLM_TENSOR_DENSE_2_OUT, "weight"), {n_embd, hparams.n_embd_out()}, TENSOR_NOT_REQUIRED);
+ dense_2_out_layers_b = create_tensor(tn(LLM_TENSOR_DENSE_2_OUT, "bias"), {hparams.n_embd_out() }, TENSOR_NOT_REQUIRED);
+
for (int i = 0; i < n_layer; ++i) {
auto & layer = layers[i];