LLM inference in C/C++
C++ ggml
convert : support Qwen3.5 embedding models (#27920)
Sam Malayek75118a3a5925
diff --git a/conversion/__init__.py b/conversion/__init__.pyindex 72960be1af..5bf4728895 100644--- a/conversion/__init__.py+++ b/conversion/__init__.py@@ -255,6 +255,7 @@ TEXT_MODEL_MAP: dict[str, str] = {"Qwen3_5ForConditionalGeneration": "qwen","Qwen3_5MoeForCausalLM": "qwen","Qwen3_5MoeForConditionalGeneration": "qwen",+ "Qwen3_5TextModel": "qwen","Qwen4ExpForCausalLM": "qwen4exp","Qwen4ExpForConditionalGeneration": "qwen4exp","RND1": "qwen",diff --git a/conversion/base.py b/conversion/base.pyindex 306a9f0f24..786e7faa19 100644--- a/conversion/base.py+++ b/conversion/base.py@@ -2336,6 +2336,16 @@ class TextModel(ModelBase):else:raise NotImplementedError("Only MEAN, CLS, and LAST pooling types supported")self.gguf_writer.add_pooling_type(pooling_type)+ else:+ embedding_config_path = self.dir_model / "embedding_config.json"+ if embedding_config_path.is_file():+ with open(embedding_config_path, encoding="utf-8") as f:+ embedding_config = json.load(f)+ pooling = embedding_config.get("pooling")+ if pooling == "last_token":+ self.gguf_writer.add_pooling_type(gguf.PoolingType.LAST)+ elif pooling is not None:+ raise NotImplementedError(f"unsupported embedding_config.json pooling {pooling!r}")# pooling before a classification head (e.g. ModernBertForSequenceClassification)if (classifier_pooling := self.hparams.get("classifier_pooling")) is not None:diff --git a/conversion/qwen.py b/conversion/qwen.pyindex 0b39187337..3af7a035cc 100644--- a/conversion/qwen.py+++ b/conversion/qwen.py@@ -650,11 +650,24 @@ class _Qwen35MRopeMixin:self.gguf_writer.add_rope_dimension_sections(self._QWEN35_DEFAULT_MROPE_SECTION)[email protected]("Qwen3_5ForConditionalGeneration", "Qwen3_5ForCausalLM")[email protected]("Qwen3_5ForConditionalGeneration", "Qwen3_5ForCausalLM", "Qwen3_5TextModel")@ModelBase.example("Qwen/Qwen3.5-9B")class Qwen3_5TextModel(_Qwen35MRopeMixin, _LinearAttentionVReorderBase):model_arch = gguf.MODEL_ARCH.QWEN35+ def __init__(self, dir_model, *args, **kwargs):+ # Inner TextModel does not own mtp.*. Set no_mtp before mixin bumps block_count.+ hparams = kwargs.pop("hparams", None)+ if hparams is None:+ hparams = ModelBase.load_hparams(dir_model, self.is_mistral_format)+ if get_model_architecture(hparams, ModelType.TEXT) == "Qwen3_5TextModel":+ self.no_mtp = True+ super().__init__(dir_model, *args, hparams=hparams, **kwargs)++ def set_gguf_parameters(self):+ super().set_gguf_parameters()+ self._try_set_pooling_type()+def _is_openjev_checkpoint(dir_model: Path) -> bool:return (dir_model / "helper" / "shim.py").is_file() and (dir_model / "config.json").is_file()diff --git a/gguf-py/gguf/tensor_mapping.py b/gguf-py/gguf/tensor_mapping.pyindex a45e8e7d3d..566b7348a7 100644--- a/gguf-py/gguf/tensor_mapping.py+++ b/gguf-py/gguf/tensor_mapping.py@@ -255,6 +255,7 @@ class TensorNameMap:"model.layers.{bid}.self_attn.language_expert_query_key_value", # cogvlm"model.layers.{bid}.linear_attn.in_proj_qkv", # qwen3.5"head.layers.{bid}.self_attn.in_proj", # laya+ "layers.{bid}.linear_attn.in_proj_qkv", # qwen3.5 text),# Attention query@@ -397,6 +398,7 @@ class TensorNameMap:MODEL_TENSOR.ATTN_GATE: ("model.layers.{bid}.self_attn.gate_proj", # afmoe muse-glimmer"model.layers.{bid}.linear_attn.in_proj_z", # qwen3.5+ "layers.{bid}.linear_attn.in_proj_z", # qwen3.5 text"model.layers.{bid}.self_attn.g_proj", # step3.5 head-wise attention gate"model.layers.{bid}.self_attn.output_gate", # minimax-01"model.layers.{bid}.self_attn.linear_gate", # hy-v4@@ -842,6 +844,7 @@ class TensorNameMap:"model.layers.{bid}.mamba.conv1d", # jamba falcon-h1 granite-hybrid"model.layers.layers.{bid}.mixer.conv1d", # plamo2"model.layers.{bid}.linear_attn.conv1d", # qwen3next+ "layers.{bid}.linear_attn.conv1d", # qwen3.5 text),MODEL_TENSOR.SSM_X: (@@ -857,6 +860,7 @@ class TensorNameMap:"model.layers.{bid}.mamba.dt_proj", # jamba falcon-h1 granite-hybrid"model.layers.layers.{bid}.mixer.dt_proj", # plamo2"model.layers.{bid}.linear_attn.dt_proj", # qwen3next+ "layers.{bid}.linear_attn.dt_proj", # qwen3.5 text"backbone.layers.{bid}.mixer.dt", # nemotron-h-moe"model.layers.{bid}.self_attn.dt_proj", # kimi"model.layers.{bid}.attention.dt_proj", # bailingmoe3@@ -873,6 +877,7 @@ class TensorNameMap:"model.layers.{bid}.mamba.A_log", # jamba falcon-h1 granite-hybrid"model.layers.layers.{bid}.mixer.A_log", # plamo2"model.layers.{bid}.linear_attn.A_log", # qwen3next+ "layers.{bid}.linear_attn.A_log", # qwen3.5 text"model.layers.{bid}.self_attn.A_log", # kimi"model.layers.{bid}.attention.A_log", # bailingmoe3),@@ -899,6 +904,7 @@ class TensorNameMap:MODEL_TENSOR.SSM_NORM: ("model.layers.{bid}.mamba.norm", # falcon-h1 granite-hybrid"model.layers.{bid}.linear_attn.norm", # qwen3next+ "layers.{bid}.linear_attn.norm", # qwen3.5 text"backbone.layers.{bid}.mixer.norm", # mamba2"model.layers.{bid}.self_attn.o_norm", # kimi"model.layers.{bid}.attention.o_norm", # bailingmoe3@@ -909,11 +915,13 @@ class TensorNameMap:"backbone.layers.{bid}.mixer.out_proj", # mamba"model.layers.{bid}.mamba.out_proj", # jamba falcon-h1 granite-hybrid"model.layers.{bid}.linear_attn.out_proj", # qwen3next+ "layers.{bid}.linear_attn.out_proj", # qwen3.5 text"model.layers.layers.{bid}.mixer.out_proj", # plamo2),MODEL_TENSOR.SSM_ALPHA: ("model.layers.{bid}.linear_attn.in_proj_a", # qwen3.5+ "layers.{bid}.linear_attn.in_proj_a", # qwen3.5 text),MODEL_TENSOR.SSM_BETA_ALPHA: (@@ -941,6 +949,7 @@ class TensorNameMap:),MODEL_TENSOR.SSM_BETA: ("model.layers.{bid}.linear_attn.in_proj_b", # qwen3.5+ "layers.{bid}.linear_attn.in_proj_b", # qwen3.5 text"model.layers.{bid}.self_attn.b_proj", # Kimi Linear"model.layers.{bid}.attention.b_proj", # bailingmoe3),