PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度
"""Comprehensive unit tests for GGUF conversion modules with low coverage.
Tests conversion modules that have 0% coverage in the test suite, covering
the actual logic paths in each module.
"""
import tempfile
from pathlib import Path
from unittest.mock import MagicMock, patch
import pytest
import torch
# ==============================================================================
# Helper: build a minimal mock model that exercises specific methods
# ==============================================================================
def _make_mock_model(cls, hparams=None):
"""Create a bare-minimum mock of a conversion model class for testing."""
if hparams is None:
hparams = {}
with patch.object(cls, "__init__", lambda self, *args, **kwargs: None):
obj = cls.__new__(cls)
obj.hparams = dict(hparams)
obj.gguf_writer = MagicMock()
obj.ftype = MagicMock()
obj.dir_model = Path(tempfile.mkdtemp())
obj.block_count = hparams.get("num_hidden_layers", 2)
obj.model_tensors = {}
obj._experts = None
obj.lerp_weights = {}
obj.lora_needs_transpose = True
rope_parameters = hparams.get("rope_parameters", hparams.get("rope_scaling")) or {}
obj.rope_parameters = dict(rope_parameters) if isinstance(rope_parameters, dict) else {}
partial_rotary_factor = (
hparams.get("partial_rotary_factor") or hparams.get("rope_pct") or hparams.get("rope_percent")
)
original_max_position_embeddings = hparams.get("original_max_position_embeddings")
rope_theta = hparams.get(
"global_rope_theta",
hparams.get(
"rope_global_theta",
hparams.get("rope_theta_global", hparams.get("rope_theta", hparams.get("rotary_emb_base"))),
),
)
local_rope_theta = hparams.get(
"local_rope_theta",
hparams.get(
"rope_local_theta",
hparams.get("rope_theta_local", hparams.get("swa_rope_theta", hparams.get("rope_local_base_freq"))),
),
)
if "full_attention" not in obj.rope_parameters and "sliding_attention" not in obj.rope_parameters:
if local_rope_theta is not None:
obj.rope_parameters["sliding_attention"] = {"rope_theta": local_rope_theta}
if "rope_theta" not in obj.rope_parameters and rope_theta is not None:
obj.rope_parameters["rope_theta"] = rope_theta
if "rope_type" not in obj.rope_parameters and obj.rope_parameters.get("type") is not None:
obj.rope_parameters["rope_type"] = obj.rope_parameters["type"]
if "partial_rotary_factor" not in obj.rope_parameters and partial_rotary_factor is not None:
obj.rope_parameters["partial_rotary_factor"] = partial_rotary_factor
if (
"original_max_position_embeddings" not in obj.rope_parameters
and original_max_position_embeddings is not None
):
obj.rope_parameters["original_max_position_embeddings"] = original_max_position_embeddings
obj.fuse_gate_up_exps = False
obj.hparams_vision = hparams.get("hparams_vision")
obj.global_config = hparams.get("global_config", {})
obj.preprocessor_config = hparams.get("preprocessor_config", {})
obj.is_mistral_format = False
obj.origin_hf_arch = None
obj.hf_arch = ""
obj.undo_permute = True
obj._is_nvfp4 = False
obj._is_mxfp4 = False
obj.head_dim = None
obj.shared_token_embeddings_found = False
obj.is_moe = False
# tensor_map needs to handle both map_tensor_name calls (key=..., try_suffixes=...)
# and format_tensor_name calls (key=..., bid=..., suffix=...)
# It also needs a .mapping attribute that returns tuples of (name, formatted_name)
def mock_get_name(key=None, try_suffixes=None):
if key is not None and try_suffixes is not None:
return key
return key or "tensor"
def mock_format_name(key, bid=None, suffix=".weight"):
if hasattr(key, "name"):
return key.name + suffix
return f"tensor_{bid if bid is not None else '0'}{suffix}"
def make_mapping():
# tensor_map.mapping is a dict-like object. When iterating over .values(),
# it yields (key, name) tuples. The unpacking is: for _, s in .values()
# So we need a dict where values are tuples of (key, name_str)
return {"tensor.weight": ("tensor_key", "tensor_name_weight")}
mock_map = MagicMock()
mock_map.get_name.side_effect = mock_get_name
mock_map.mapping = make_mapping()
obj.tensor_map = mock_map
obj.format_tensor_name = mock_format_name
return obj
@pytest.mark.parametrize("disable_mtp_export", [False, True])
def test_qwen35_gguf_mtp_export_control(monkeypatch, tmp_path, disable_mtp_export):
from auto_round.export.export_to_gguf import export
from auto_round.export.export_to_gguf.config import ModelType
from auto_round.export.export_to_gguf.conversion.base import ModelBase as ConversionModelBase
from auto_round.export.export_to_gguf.conversion.qwen import Qwen3_5TextModel
hparams = {
"architectures": ["Qwen3_5ForConditionalGeneration"],
"num_hidden_layers": 32,
"mtp_num_hidden_layers": 0 if disable_mtp_export else 1,
}
class FakeModel:
name_or_path = str(tmp_path)
config = type("Config", (), {"model_type": "qwen3_5"})()
class FakeConversion:
ModelBase = ConversionModelBase
@staticmethod
def model_type(model_type):
return model_type
@staticmethod
def get_model_architecture(_hparams, _model_type):
return "Qwen3_5ForConditionalGeneration"
@staticmethod
def get_model_class(_model_architecture, model_type):
return Qwen3_5TextModel
monkeypatch.setattr(ConversionModelBase, "load_hparams", staticmethod(lambda *_args: hparams.copy()))
monkeypatch.setattr(export, "get_conversion", lambda *_args, **_kwargs: FakeConversion)
monkeypatch.setattr(export, "wrapper_model_instance", lambda model_instance, **_kwargs: model_instance)
monkeypatch.setattr(export, "handle_special_model", lambda model_instance, _architecture: model_instance)
if disable_mtp_export:
monkeypatch.setenv("AR_DISABLE_GGUF_MTP_EXPORT", "1")
else:
monkeypatch.delenv("AR_DISABLE_GGUF_MTP_EXPORT", raising=False)
model = export.create_model_class(tmp_path, FakeModel(), {}, model_type=ModelType.TEXT)
item = ("mtp.layers.0.mlp.down_proj.weight", lambda: torch.ones(1, 1))
filtered = model.filter_tensors(item)
if disable_mtp_export:
assert model.block_count == 32
assert filtered is None
assert Qwen3_5TextModel.no_mtp is False
return
assert filtered is not None
name, tensor_fn = filtered
outputs = list(model.modify_tensors(tensor_fn(), name, bid=32))
assert model.block_count == 33
assert len(outputs) == 1
assert outputs[0][0] == "blk.32.ffn_down.weight"
assert torch.equal(outputs[0][1], tensor_fn())
# ==============================================================================
# granite.py tests
# ==============================================================================
class TestGraniteMoeConversion:
"""Tests for aggregated expert tensors restored from linearized Granite MoE modules."""
@staticmethod
def _make_model(hparams, architecture="GraniteMoeHybridForCausalLM"):
from auto_round.export.export_to_gguf.conversion.granite import GraniteMoeModel
from auto_round.export.export_to_gguf.special_handle import handle_special_model
obj = _make_mock_model(GraniteMoeModel, hparams)
return handle_special_model(obj, architecture)
@pytest.mark.parametrize(
("projection", "tensor_type"),
[
("down_proj", "FFN_DOWN_EXP"),
("gate_proj", "FFN_GATE_EXP"),
("up_proj", "FFN_UP_EXP"),
],
)
@pytest.mark.parametrize("suffix", ["", ".weight"])
def test_modify_tensors_maps_aggregated_expert_projection(self, projection, tensor_type, suffix):
obj = self._make_model({"num_local_experts": 4})
data = torch.randn(4, 8, 16)
name = f"model.layers.0.block_sparse_moe.experts.{projection}{suffix}"
result = list(obj.modify_tensors(data, name, bid=0))
assert result == [(f"{tensor_type}.weight", data)]
def test_modify_tensors_rejects_non_aggregated_expert_projection(self):
obj = self._make_model({"num_local_experts": 4})
with pytest.raises(ValueError, match="three-dimensional"):
list(
obj.modify_tensors(
torch.randn(8, 16),
"model.layers.0.block_sparse_moe.experts.down_proj",
bid=0,
)
)
@pytest.mark.parametrize("suffix", ["", ".weight"])
def test_modify_tensors_splits_aggregated_gate_up_projection(self, suffix):
obj = self._make_model({"num_local_experts": 4, "intermediate_size": 8})
data = torch.randn(4, 16, 32)
name = f"model.layers.0.block_sparse_moe.experts.gate_up_proj{suffix}"
result = list(obj.modify_tensors(data, name, bid=0))
assert [item[0] for item in result] == ["FFN_GATE_EXP.weight", "FFN_UP_EXP.weight"]
assert [item[1].shape for item in result] == [(4, 8, 32), (4, 8, 32)]
# ==============================================================================
# mimo.py tests
# ==============================================================================
class TestMimoConversion:
"""Tests for MiMo conversion module."""
def test_tp_aware_qkv_dequant_tp4(self):
"""Test _tp_aware_qkv_dequant with TP=4 configuration."""
from auto_round.export.export_to_gguf.conversion.mimo import MimoV2Model
# n_q=8, n_kv=2, hd=64, vhd=64
# q_size=512, k_size=128, v_size=128, total=768
n_q, n_kv, hd, vhd = 8, 2, 64, 64
total_rows = n_q * hd + n_kv * hd + n_kv * vhd # 768
n_col = 1024
bs = 128
weight = torch.randn(total_rows, n_col)
# TP=4: total_rows % 4 == 0, rows_per_rank = 192, bpr = ceil(192/128) = 2
# scale_inv shape = tp * bpr x n_col_blocks = 4*2 x ceil(1024/128)=8
# n_col_blocks = ceil(n_col/bs) for proper broadcasting
n_col_blocks = (n_col + bs - 1) // bs # = 8
scale_inv = torch.randn(8, n_col_blocks)
result = MimoV2Model._tp_aware_qkv_dequant(weight, scale_inv, n_q, n_kv, hd, vhd, bs=bs)
assert result.shape == (total_rows, n_col)
def test_tp_aware_qkv_dequant_tp8(self):
"""Test _tp_aware_qkv_dequant with TP=8 configuration."""
from auto_round.export.export_to_gguf.conversion.mimo import MimoV2Model
n_q, n_kv, hd, vhd = 8, 2, 64, 64
total_rows = n_q * hd + n_kv * hd + n_kv * vhd # 768
n_col = 1024
bs = 128
weight = torch.randn(total_rows, n_col)
# TP=8: total_rows % 8 == 0, rows_per_rank = 96, bpr = ceil(96/128) = 1
# scale_inv shape = tp * bpr x n_col_blocks = 8*1 x 8
n_col_blocks = (n_col + bs - 1) // bs # = 8
scale_inv = torch.randn(8, n_col_blocks)
result = MimoV2Model._tp_aware_qkv_dequant(weight, scale_inv, n_q, n_kv, hd, vhd, bs=bs)
assert result.shape == (total_rows, n_col)
def test_tp_aware_qkv_dequant_invalid_rows(self):
"""Test that mismatched weight rows raise ValueError."""
from auto_round.export.export_to_gguf.conversion.mimo import MimoV2Model
weight = torch.randn(100, 64)
scale_inv = torch.randn(2, 1)
with pytest.raises(ValueError, match="qkv_proj weight rows"):
MimoV2Model._tp_aware_qkv_dequant(weight, scale_inv, 4, 2, 32, 32)
def test_tp_aware_qkv_dequant_cannot_detect_tp(self):
"""Test that undetectable TP raises ValueError."""
from auto_round.export.export_to_gguf.conversion.mimo import MimoV2Model
n_q, n_kv, hd, vhd = 8, 2, 64, 64
total_rows = n_q * hd + n_kv * hd + n_kv * vhd
weight = torch.randn(total_rows, 512)
scale_inv = torch.randn(7, 1) # no candidate TP matches
with pytest.raises(ValueError, match="cannot detect TP"):
MimoV2Model._tp_aware_qkv_dequant(weight, scale_inv, n_q, n_kv, hd, vhd)
def test_filter_tensors_attention_sink_without_weight_suffix(self):
"""Test filter_tensors appends .weight to attention_sink tensor name."""
from auto_round.export.export_to_gguf.conversion.mimo import MimoV2Model
obj = _make_mock_model(MimoV2Model)
obj.filter_tensors = MimoV2Model.filter_tensors.__get__(obj, MimoV2Model)
name, gen = "model.layers.0.attention_sink", lambda: None
result = obj.filter_tensors((name, gen))
assert result is not None
assert result[0] == "model.layers.0.attention_sink.weight"
def test_filter_tensors_attention_sink_with_weight_suffix(self):
"""Test filter_tensors leaves attention_sink.weight unchanged."""
from auto_round.export.export_to_gguf.conversion.mimo import MimoV2Model
obj = _make_mock_model(MimoV2Model)
obj.filter_tensors = MimoV2Model.filter_tensors.__get__(obj, MimoV2Model)
name, gen = "model.layers.0.attention_sink.weight", lambda: None
result = obj.filter_tensors((name, gen))
assert result is not None
assert result[0] == "model.layers.0.attention_sink.weight"
def test_prepare_tensors_unprocessed_experts_error(self):
"""Test that unprocessed experts raise ValueError."""
from auto_round.export.export_to_gguf.conversion.mimo import MimoV2Model
obj = _make_mock_model(
MimoV2Model,
{
"num_hidden_layers": 2,
"n_routed_experts": 4,
},
)
obj._experts = [{"unprocessed.tensor.0": None}]
obj.tensor_map.mapping = {"tensor": ("KEY", "tensor_name")}
# Mock super().prepare_tensors() to skip the base class work
with (
patch("auto_round.export.export_to_gguf.conversion.base.ModelBase.prepare_tensors"),
pytest.raises(ValueError, match="Unprocessed experts"),
):
obj.prepare_tensors()
# ==============================================================================
# minicpm.py tests
# ==============================================================================
class TestMiniCPMConversion:
"""Tests for MiniCPM conversion module."""
def test_generate_extra_tensors_with_rope_scaling(self):
"""Test generate_extra_tensors yields rope long/short factors."""
from auto_round.export.export_to_gguf.conversion.minicpm import MiniCPMModel
rope_dims = 64
long_factors = [1.0] * (rope_dims // 2)
short_factors = [1.0] * (rope_dims // 2)
obj = _make_mock_model(
MiniCPMModel,
{
"num_hidden_layers": 2,
"hidden_size": 2048,
"num_attention_heads": 32,
"scale_emb": 1.0,
"scale_depth": 4.0,
"dim_model_base": 1024,
"rope_scaling": {
"long_factor": long_factors,
"short_factor": short_factors,
},
},
)
results = list(obj.generate_extra_tensors())
assert len(results) == 2
# The tensor name contains ROPE_FACTORS_LONG
assert "ROPE_FACTORS_LONG" in results[0][0]
assert "ROPE_FACTORS_SHORT" in results[1][0]
assert results[0][1].shape == (rope_dims // 2,)
def test_generate_extra_tensors_missing_long_factor_raises(self):
"""Test missing long_factor raises KeyError."""
from auto_round.export.export_to_gguf.conversion.minicpm import MiniCPMModel
obj = _make_mock_model(
MiniCPMModel,
{
"num_hidden_layers": 2,
"hidden_size": 2048,
"num_attention_heads": 32,
"scale_emb": 1.0,
"scale_depth": 4.0,
"dim_model_base": 1024,
"rope_scaling": {
"short_factor": [1.0] * 32,
},
},
)
with pytest.raises(KeyError, match="long_factor"):
list(obj.generate_extra_tensors())
def test_generate_extra_tensors_length_mismatch_raises(self):
"""Test mismatched factor lengths raise ValueError."""
from auto_round.export.export_to_gguf.conversion.minicpm import MiniCPMModel
obj = _make_mock_model(
MiniCPMModel,
{
"num_hidden_layers": 2,
"hidden_size": 2048,
"num_attention_heads": 32,
"scale_emb": 1.0,
"scale_depth": 4.0,
"dim_model_base": 1024,
"rope_scaling": {
"long_factor": [1.0] * 64,
"short_factor": [1.0] * 32, # wrong length
},
},
)
with pytest.raises(ValueError, match="length of rope long and short factors"):
list(obj.generate_extra_tensors())
def test_minicpm3_reverse_hf_permute(self):
"""Test MiniCPM3Model._reverse_hf_permute transforms tensor correctly."""
from auto_round.export.export_to_gguf.conversion.minicpm import MiniCPM3Model
obj = _make_mock_model(
MiniCPM3Model,
{
"num_hidden_layers": 2,
"num_attention_heads": 8,
"num_key_value_heads": 2,
"qk_nope_head_dim": 64,
"qk_rope_head_dim": 32,
"v_head_dim": 64,
"kv_lora_rank": 128,
"hidden_size": 512,
"rms_norm_eps": 1e-5,
"max_position_embeddings": 4096,
},
)
tensor = torch.randn(256, 512)
result = obj._reverse_hf_permute(tensor, 4, 2)
assert result.shape == tensor.shape
def test_minicpm3_reverse_hf_permute_same_head(self):
"""Test _reverse_hf_permute when n_kv_head equals n_head."""
from auto_round.export.export_to_gguf.conversion.minicpm import MiniCPM3Model
obj = _make_mock_model(
MiniCPM3Model,
{
"num_hidden_layers": 2,
"num_attention_heads": 8,
"num_key_value_heads": 8,
"qk_nope_head_dim": 64,
"qk_rope_head_dim": 64,
"v_head_dim": 64,
"kv_lora_rank": 128,
"hidden_size": 512,
"rms_norm_eps": 1e-5,
"max_position_embeddings": 4096,
},
)
tensor = torch.randn(512, 512)
result = obj._reverse_hf_permute(tensor, 8, 8)
assert result.shape == tensor.shape
# ==============================================================================
# minimax.py tests
# ==============================================================================
class TestMiniMaxConversion:
"""Tests for MiniMax conversion module."""
def test_set_gguf_parameters(self):
"""Test MiniMaxM2Model.set_gguf_parameters sets MoE parameters."""
from auto_round.export.export_to_gguf.conversion.minimax import MiniMaxM2Model
obj = _make_mock_model(
MiniMaxM2Model,
{
"num_hidden_layers": 2,
"num_local_experts": 8,
"intermediate_size": 10944,
"rotary_dim": 32,
"num_attention_heads": 8,
"num_key_value_heads": 2,
"hidden_size": 2048,
},
)
obj.set_gguf_parameters()
# Calls add_expert_feed_forward_length and add_rope_dimension_count
obj.gguf_writer.add_expert_feed_forward_length.assert_called_with(10944)
obj.gguf_writer.add_rope_dimension_count.assert_called_with(32)
# ==============================================================================
# mpt.py tests
# ==============================================================================
class TestMPTConversion:
"""Tests for MPT conversion module."""
def test_modify_tensors_with_scales(self):
"""Test modify_tensors handles names containing 'scales'."""
from auto_round.export.export_to_gguf.conversion.mpt import MPTModel
obj = _make_mock_model(
MPTModel,
{
"num_hidden_layers": 2,
"max_seq_len": 2048,
"d_model": 2048,
"n_heads": 16,
"attn_config": {"kv_n_heads": 4, "clip_qkv": None, "alibi": False, "alibi_bias_max": 8.0},
},
)
data = torch.randn(2048, 2048)
# The MPT model replaces "scales" -> "act.scales" in map_tensor_name result
# Our mock returns the key as-is, so we check it doesn't crash
results = list(obj.modify_tensors(data, "blk.0.ffn.scales", bid=0))
assert len(results) == 1
def test_modify_tensors_regular_weight(self):
"""Test regular weight tensor is mapped normally."""
from auto_round.export.export_to_gguf.conversion.mpt import MPTModel
obj = _make_mock_model(
MPTModel,
{
"num_hidden_layers": 2,
"max_seq_len": 2048,
"d_model": 2048,
"n_heads": 16,
"attn_config": {"kv_n_heads": 4, "clip_qkv": None, "alibi": False, "alibi_bias_max": 8.0},
},
)
data = torch.randn(2048, 2048)
obj.tensor_map.get_name.return_value = "blk.0.attn_q.weight"
results = list(obj.modify_tensors(data, "blk.0.attn_q.weight", bid=0))
assert len(results) == 1
# ==============================================================================
# nemotron.py tests
# ==============================================================================
class TestNemotronConversion:
"""Tests for Nemotron conversion module."""
def test_nemotron_modify_tensors_norm_weight_plus_one(self):
"""Test that norm.weight tensors get +1 added."""
from auto_round.export.export_to_gguf.conversion.nemotron import NemotronModel
obj = _make_mock_model(
NemotronModel,
{
"num_hidden_layers": 2,
"vocab_size": 32000,
"hidden_size": 2048,
"intermediate_size": 8192,
"num_attention_heads": 16,
"num_key_value_heads": 16,
"partial_rotary_factor": 0.25,
"layer_norm_eps": 1e-5,
"rope_pct": 0.25,
},
)
data = torch.ones(2048) * 0.5
results = list(obj.modify_tensors(data, "model.layers.0.input_layernorm.weight", bid=0))
assert len(results) == 1
assert torch.allclose(results[0][1], torch.ones(2048) * 1.5)
def test_nemotron_modify_tensors_non_norm_unchanged(self):
"""Test non-norm tensors are passed through unchanged."""
from auto_round.export.export_to_gguf.conversion.nemotron import NemotronModel
obj = _make_mock_model(
NemotronModel,
{
"num_hidden_layers": 2,
"vocab_size": 32000,
"hidden_size": 2048,
"intermediate_size": 8192,
"num_attention_heads": 16,
"num_key_value_heads": 16,
"partial_rotary_factor": 0.25,
"layer_norm_eps": 1e-5,
"rope_pct": 0.25,
},
)
data = torch.randn(2048, 2048)
original = data.clone()
results = list(obj.modify_tensors(data, "model.layers.0.self_attn.q_proj.weight", bid=0))
assert torch.equal(results[0][1], original)
def test_nemotron_set_gguf_parameters_rope_scaling_linear(self):
"""Test rope_scaling with LINEAR type."""
from auto_round.export.export_to_gguf.conversion.nemotron import NemotronModel
obj = _make_mock_model(
NemotronModel,
{
"num_hidden_layers": 2,
"vocab_size": 32000,
"hidden_size": 2048,
"intermediate_size": 8192,
"num_attention_heads": 16,
"num_key_value_heads": 16,
"partial_rotary_factor": 0.25,
"layer_norm_eps": 1e-5,
"rope_pct": 0.25,
"rope_scaling": {"type": "linear"},
"factor": 4.0,
},
)
obj.rope_parameters = {"rope_type": "linear", "factor": 4.0, "partial_rotary_factor": 0.25}
obj.set_gguf_parameters()
obj.gguf_writer.add_rope_scaling_type.assert_called()
obj.gguf_writer.add_rope_scaling_factor.assert_called_with(4.0)
def test_nemotron_set_gguf_parameters_rope_scaling_none(self):
"""Test when rope_scaling is None."""
from auto_round.export.export_to_gguf.conversion.nemotron import NemotronModel
obj = _make_mock_model(
NemotronModel,
{
"num_hidden_layers": 2,
"vocab_size": 32000,
"hidden_size": 2048,
"intermediate_size": 8192,
"num_attention_heads": 16,
"num_key_value_heads": 16,
"partial_rotary_factor": 0.25,
"layer_norm_eps": 1e-5,
"rope_pct": 0.25,
"rope_scaling": None,
},
)
obj.set_gguf_parameters()
obj.gguf_writer.add_rope_scaling_type.assert_called()
def test_nemotron_nanov2_filter_tensors_input_conditioner(self):
"""Test NemotronNanoV2VLModel.filter_tensors skips input_conditioner."""
from auto_round.export.export_to_gguf.conversion.nemotron import NemotronNanoV2VLModel
obj = _make_mock_model(
NemotronNanoV2VLModel,
{
"vision_config": {"ImageSize": 512},
},
)
obj.hparams_vision = {"patch_size": 14}
obj.global_config = {"force_image_size": 512, "vision_config": {}}
obj.preprocessor_config = {}
# Should skip input_conditioner
result = obj.filter_tensors(("input_conditioner.some_tensor", lambda: None))
assert result is None
def test_nemotron_nanov2_filter_tensors_video_skip(self):
"""Test NemotronNanoV2VLModel.filter_tensors skips video tensors."""
from auto_round.export.export_to_gguf.conversion.nemotron import NemotronNanoV2VLModel
obj = _make_mock_model(
NemotronNanoV2VLModel,
{
"vision_config": {"ImageSize": 512},
},
)
obj.hparams_vision = {"patch_size": 14}
obj.global_config = {"force_image_size": 512, "vision_config": {}}
obj.preprocessor_config = {}
# Should skip video tensors
result = obj.filter_tensors(
("vision_model.radio_model.model.patch_generator.video_embedder.tensor", lambda: None)
)
assert result is None
def test_nemotron_nanov2_filter_tensors_passes_vision(self):
"""Test NemotronNanoV2VLModel.filter_tensors passes vision tensors."""
from auto_round.export.export_to_gguf.conversion.nemotron import NemotronNanoV2VLModel
obj = _make_mock_model(
NemotronNanoV2VLModel,
{
"vision_config": {"ImageSize": 512},
},
)
obj.hparams_vision = {"patch_size": 14}
obj.global_config = {"force_image_size": 512, "vision_config": {}}
obj.preprocessor_config = {}
result = obj.filter_tensors(("vision_model.radio_model.model.patch_generator.pos_embed", lambda: None))
assert result is not None
# ==============================================================================
# olmo.py tests
# ==============================================================================
class TestOlmoConversion:
"""Tests for Olmo conversion module."""
def test_olmo_modify_tensors_q_proj_permute(self):
"""Test OlmoModel permutes q_proj tensor."""
from auto_round.export.export_to_gguf.conversion.olmo import OlmoModel
obj = _make_mock_model(
OlmoModel,
{
"num_hidden_layers": 2,
"num_attention_heads": 8,
"num_key_value_heads": 8,
"hidden_size": 2048,
},
)
n_head, hd, hidden = 8, 64, 2048
tensor = torch.randn(n_head * hd, hidden)
results = list(obj.modify_tensors(tensor, "blk.0.self_attn.q_proj.weight", bid=0))
assert len(results) == 1
assert results[0][1].shape == tensor.shape
def test_olmo2_set_gguf_parameters_sliding_window(self):
"""Test Olmo2Model adds sliding window pattern."""
from auto_round.export.export_to_gguf.conversion.olmo import Olmo2Model
obj = _make_mock_model(
Olmo2Model,
{
"num_hidden_layers": 8,
"hidden_size": 2048,
"sliding_window": 4096,
"layer_types": ["sliding_attention", "full_attention", "sliding_attention", "full_attention"],
},
)
obj.set_gguf_parameters()
obj.gguf_writer.add_sliding_window.assert_called_with(4096)
obj.gguf_writer.add_sliding_window_pattern.assert_called()
def test_olmo2_set_gguf_parameters_no_layer_types(self):
"""Test Olmo2Model with no layer_types defaults to every-4th."""
from auto_round.export.export_to_gguf.conversion.olmo import Olmo2Model
obj = _make_mock_model(
Olmo2Model,
{
"num_hidden_layers": 8,
"hidden_size": 2048,
"sliding_window": 4096,
},
)
obj.set_gguf_parameters()
obj.gguf_writer.add_sliding_window_pattern.assert_called()
call_args = obj.gguf_writer.add_sliding_window_pattern.call_args[0][0]
assert len(call_args) == 8
def test_olmoe_prepare_tensors_unprocessed_experts_error(self):
"""Test unprocessed experts raise ValueError."""
from auto_round.export.export_to_gguf.conversion.olmo import OlmoeModel
obj = _make_mock_model(
OlmoeModel,
{
"num_hidden_layers": 2,
"num_local_experts": 8,
"hidden_size": 2048,
},
)
obj._experts = [{"unprocessed.tensor": None}]
obj.tensor_map.mapping = {"tensor": ("KEY", "tensor_name")}
with (
patch("auto_round.export.export_to_gguf.conversion.base.ModelBase.prepare_tensors"),
pytest.raises(ValueError, match="Unprocessed experts"),
):
obj.prepare_tensors()
# ==============================================================================
# openelm.py tests
# ==============================================================================
class TestOpenELMConversion:
"""Tests for OpenELM conversion module."""
def test_find_hparam_n_layers(self):
"""Test find_hparam returns num_transformer_layers for n_layers key."""
from auto_round.export.export_to_gguf.conversion.openelm import OpenELMModel
obj = _make_mock_model(
OpenELMModel,
{
"num_transformer_layers": 12,
"model_dim": 1024,
"ffn_multipliers": [2.0] * 12,
"ffn_dim_divisor": 64,
"num_kv_heads": [2] * 12,
"num_query_heads": [4] * 12,
"head_dim": 128,
"vocab_size": 32000,
"max_context_length": 2048,
"rope_freq_constant": 10000.0,
},
)
obj._n_embd = 1024
obj._num_kv_heads = [2] * 12
obj._num_query_heads = [4] * 12
obj._ffn_dims = [2048] * 12
result = obj.find_hparam(["n_layers"])
assert result == 12
def test_modify_tensors_ffn_split(self):
"""Test OpenELMModel splits ffn.proj_1.weight into gate and up."""
from auto_round.export.export_to_gguf.conversion.openelm import OpenELMModel
obj = _make_mock_model(
OpenELMModel,
{
"num_transformer_layers": 2,
"model_dim": 1024,
"ffn_multipliers": [2.0, 2.0],
"ffn_dim_divisor": 64,
"num_kv_heads": [2, 2],
"num_query_heads": [4, 4],
"head_dim": 128,
"vocab_size": 32000,
"max_context_length": 2048,
"rope_freq_constant": 10000.0,
},
)
obj._n_embd = 1024
obj._num_kv_heads = [2, 2]
obj._num_query_heads = [4, 4]
obj._ffn_dims = [2048, 2048]
# Use bid=1 to avoid index out of range in ffn_dims access
tensor = torch.randn(2048, 1024)
results = list(obj.modify_tensors(tensor, "transformer.layers.1.ffn.proj_1.weight", bid=1))
# Should yield two tensors: FFN_GATE and FFN_UP
assert len(results) == 2
# ==============================================================================
# orion.py tests
# ==============================================================================
class TestOrionConversion:
"""Tests for Orion conversion module."""
def test_set_gguf_parameters_max_sequence_length(self):
"""Test context length from max_sequence_length."""
from auto_round.export.export_to_gguf.conversion.orion import OrionModel
obj = _make_mock_model(
OrionModel,
{
"num_hidden_layers": 2,
"hidden_size": 4096,
"intermediate_size": 11008,
"num_attention_heads": 32,
"max_sequence_length": 8192,
"rms_norm_eps": 1e-6,
},
)
obj.set_gguf_parameters()
obj.gguf_writer.add_context_length.assert_called_with(8192)
def test_set_gguf_parameters_max_position_embeddings(self):
"""Test context length from max_position_embeddings."""
from auto_round.export.export_to_gguf.conversion.orion import OrionModel
obj = _make_mock_model(
OrionModel,
{
"num_hidden_layers": 2,
"hidden_size": 4096,
"intermediate_size": 11008,
"num_attention_heads": 32,
"max_position_embeddings": 4096,
"rms_norm_eps": 1e-6,
},
)
obj.set_gguf_parameters()
obj.gguf_writer.add_context_length.assert_called_with(4096)
def test_set_gguf_parameters_model_max_length(self):
"""Test context length from model_max_length."""
from auto_round.export.export_to_gguf.conversion.orion import OrionModel
obj = _make_mock_model(
OrionModel,
{
"num_hidden_layers": 2,
"hidden_size": 4096,
"intermediate_size": 11008,
"num_attention_heads": 32,
"model_max_length": 16384,
"rms_norm_eps": 1e-6,
},
)
obj.set_gguf_parameters()
obj.gguf_writer.add_context_length.assert_called_with(16384)
def test_set_gguf_parameters_raises_without_ctx_length(self):
"""Test ValueError when no context length parameter is present."""
from auto_round.export.export_to_gguf.conversion.orion import OrionModel
obj = _make_mock_model(
OrionModel,
{
"num_hidden_layers": 2,
"hidden_size": 4096,
"intermediate_size": 11008,
"num_attention_heads": 32,
"rms_norm_eps": 1e-6,
},
)
with pytest.raises(ValueError, match="can not find ctx length"):
obj.set_gguf_parameters()
from auto_round.export.export_to_gguf.conversion.orion import OrionModel
obj = _make_mock_model(OrionModel)
with patch.object(obj, "_set_vocab_sentencepiece") as mock:
obj.set_vocab()
mock.assert_called_once_with()
def test_set_gguf_parameters_with_max_sequence_length(self):
"""Test set_gguf_parameters picks the right context length key."""
from auto_round.export.export_to_gguf.conversion.orion import OrionModel
obj = _make_mock_model(
OrionModel,
{
"num_attention_heads": 16,
"max_sequence_length": 8192,
"hidden_size": 4096,
"intermediate_size": 16384,
"rms_norm_eps": 1e-5,
},
)
obj.set_gguf_parameters()
obj.gguf_writer.add_context_length.assert_called_once_with(8192)
def test_set_gguf_parameters_with_max_position_embeddings(self):
"""Test set_gguf_parameters falls back to max_position_embeddings."""
from auto_round.export.export_to_gguf.conversion.orion import OrionModel
obj = _make_mock_model(
OrionModel,
{
"num_attention_heads": 16,
"max_position_embeddings": 4096,
"hidden_size": 4096,
"intermediate_size": 16384,
"rms_norm_eps": 1e-5,
},
)
obj.set_gguf_parameters()
obj.gguf_writer.add_context_length.assert_called_once_with(4096)
# ==============================================================================
# pangu.py tests
# ==============================================================================
class TestPanguConversion:
"""Tests for Pangu conversion module."""
def test_modify_tensors_tied_lm_head(self):
"""Test that tied lm_head.weight is skipped."""
from auto_round.export.export_to_gguf.conversion.pangu import PanguEmbeddedModel
obj = _make_mock_model(
PanguEmbeddedModel,
{
"num_hidden_layers": 2,
"hidden_size": 2048,
"num_attention_heads": 16,
"head_dim": 64,
"vocab_size": 43008,
"tie_word_embeddings": True,
},
)
data = torch.randn(43008, 2048)
results = list(obj.modify_tensors(data, "lm_head.weight", bid=None))
assert len(results) == 0
def test_modify_tensors_untied_lm_head(self):
"""Test that untied lm_head.weight is passed through."""
from auto_round.export.export_to_gguf.conversion.pangu import PanguEmbeddedModel
obj = _make_mock_model(
PanguEmbeddedModel,
{
"num_hidden_layers": 2,
"hidden_size": 2048,
"num_attention_heads": 16,
"head_dim": 64,
"vocab_size": 43008,
"tie_word_embeddings": False,
},
)
data = torch.randn(43008, 2048)
results = list(obj.modify_tensors(data, "lm_head.weight", bid=None))
assert len(results) == 1
from auto_round.export.export_to_gguf.conversion.pangu import PanguEmbeddedModel
obj = _make_mock_model(
PanguEmbeddedModel,
{
"vocab_size": 32000,
"head_dim": 128,
"hidden_size": 2048,
"num_attention_heads": 16,
"max_position_embeddings": 4096,
"intermediate_size": 8192,
},
)
with patch.object(PanguEmbeddedModel.__mro__[1], "set_gguf_parameters", lambda self: None):
obj.set_gguf_parameters()
obj.gguf_writer.add_rope_dimension_count.assert_called_once_with(128)
def test_set_gguf_parameters_without_head_dim(self):
"""Test set_gguf_parameters derives rope_dim from hidden_size/num_heads."""
from auto_round.export.export_to_gguf.conversion.pangu import PanguEmbeddedModel
obj = _make_mock_model(
PanguEmbeddedModel,
{
"vocab_size": 32000,
"hidden_size": 2048,
"num_attention_heads": 16,
"max_position_embeddings": 4096,
"intermediate_size": 8192,
},
)
with patch.object(PanguEmbeddedModel.__mro__[1], "set_gguf_parameters", lambda self: None):
obj.set_gguf_parameters()
# 2048 / 16 = 128
obj.gguf_writer.add_rope_dimension_count.assert_called_once_with(128)
obj.gguf_writer.add_key_length.assert_called_once_with(128)
obj.gguf_writer.add_value_length.assert_called_once_with(128)
# ==============================================================================
# plamo.py tests
# ==============================================================================
class TestPlamoConversion:
"""Tests for Plamo conversion module."""
def test_plamo_shuffle_attn_q_weight(self):
"""Test shuffle_attn_q_weight reshapes and permutes correctly."""
from auto_round.export.export_to_gguf.conversion.plamo import PlamoModel
obj = _make_mock_model(
PlamoModel,
{
"num_hidden_layers": 2,
"hidden_size": 5120,
"intermediate_size": 8192,
"num_attention_heads": 40,
"num_key_value_heads": 5,
"rms_norm_eps": 1e-6,
},
)
data = torch.randn(5120, 5120)
result = obj.shuffle_attn_q_weight(data)
assert result.shape == (5120, 5120)
assert not torch.equal(result, data)
def test_plamo_shuffle_attn_output_weight(self):
"""Test shuffle_attn_output_weight reshapes and permutes correctly."""
from auto_round.export.export_to_gguf.conversion.plamo import PlamoModel
obj = _make_mock_model(
PlamoModel,
{
"num_hidden_layers": 2,
"hidden_size": 5120,
"intermediate_size": 8192,
"num_attention_heads": 40,
"num_key_value_heads": 5,
"rms_norm_eps": 1e-6,
},
)
data = torch.randn(5120, 5120)
result = obj.shuffle_attn_output_weight(data)
assert result.shape == (5120, 5120)
assert not torch.equal(result, data)
def test_plamo_modify_tensors_q_weight_shuffle(self):
"""Test attn_q.weight triggers shuffle."""
from auto_round.export.export_to_gguf.conversion.plamo import PlamoModel
obj = _make_mock_model(
PlamoModel,
{
"num_hidden_layers": 2,
"hidden_size": 5120,
"intermediate_size": 8192,
"num_attention_heads": 40,
"num_key_value_heads": 5,
"rms_norm_eps": 1e-6,
},
)
data = torch.randn(5120, 5120)
results = list(obj.modify_tensors(data, "blk.0.attn_q.weight", bid=0))
assert len(results) == 1
assert results[0][1].shape == (5120, 5120)
def test_plamo2_modify_tensors_A_log_transform(self):
"""Test A_log transformation (negate exp)."""
from auto_round.export.export_to_gguf.conversion.plamo import Plamo2Model
obj = _make_mock_model(
Plamo2Model,
{
"num_hidden_layers": 2,
"hidden_size": 4096,
"intermediate_size": 8192,
"num_attention_heads": 32,
"num_key_value_heads": 4,
"vocab_size": 32000,
"rms_norm_eps": 1e-6,
"mamba_step": 2,
"mamba_enabled": True,
},
)
obj.rope_parameters = {"rope_theta": 10000}
data = torch.tensor([0.0, 1.0, 2.0])
results = list(obj.modify_tensors(data, "blk.0.mixer.A_log", bid=0))
assert torch.equal(results[0][1], -torch.exp(data))
def test_plamo2_modify_tensors_pre_mixer_norm_plus_one(self):
"""Test pre_mixer_norm.weight gets +1 added."""
from auto_round.export.export_to_gguf.conversion.plamo import Plamo2Model
obj = _make_mock_model(
Plamo2Model,
{
"num_hidden_layers": 2,
"hidden_size": 4096,
"intermediate_size": 8192,
"num_attention_heads": 32,
"num_key_value_heads": 4,
"vocab_size": 32000,
"rms_norm_eps": 1e-6,
"mamba_step": 2,
"mamba_enabled": True,
},
)
obj.rope_parameters = {"rope_theta": 10000}
data = torch.ones(4096) * 0.5
results = list(obj.modify_tensors(data, "blk.0.mixer.pre_mixer_norm.weight", bid=0))
assert torch.allclose(results[0][1], torch.ones(4096) * 1.5)
def test_plamo2_set_gguf_parameters_mamba_layers(self):
"""Test Plamo2Model with mamba layers sets head counts correctly."""
from auto_round.export.export_to_gguf.conversion.plamo import Plamo2Model
obj = _make_mock_model(
Plamo2Model,
{
"num_hidden_layers": 8,
"hidden_size": 4096,
"intermediate_size": 8192,
"num_attention_heads": 32,
"num_key_value_heads": 4,
"vocab_size": 32000,
"rms_norm_eps": 1e-6,
"mamba_step": 2,
"mamba_enabled": True,
"hidden_size_per_head": 128,
},
)
obj.rope_parameters = {"rope_theta": 10000}
obj.set_gguf_parameters()
obj.gguf_writer.add_head_count_kv.assert_called()
obj.gguf_writer.add_head_count.assert_called()
def test_plamo3_modify_tensors_norm_plus_one(self):
"""Test norm.weight gets +1 in Plamo3Model."""
from auto_round.export.export_to_gguf.conversion.plamo import Plamo3Model
obj = _make_mock_model(
Plamo3Model,
{
"num_hidden_layers": 2,
"hidden_size": 4096,
"intermediate_size": 8192,
"num_attention_heads": 32,
"num_key_value_heads": 32,
"vocab_size": 32000,
},
)
data = torch.ones(4096)
results = list(obj.modify_tensors(data, "blk.0.norm.weight", bid=0))
assert torch.allclose(results[0][1], torch.ones(4096) * 2.0)
# ==============================================================================
# rwkv.py tests
# ==============================================================================
class TestRWKVConversion:
"""Tests for RWKV conversion module."""
def test_rwkv6_set_gguf_parameters(self):
"""Test Rwkv6Model.set_gguf_parameters sets all expected parameters."""
from auto_round.export.export_to_gguf.conversion.rwkv import Rwkv6Model
obj = _make_mock_model(
Rwkv6Model,
{
"num_hidden_layers": 2,
"head_size": 64,
"hidden_size": 4096,
"intermediate_size": 28672,
"layer_norm_epsilon": 1e-5,
"rescale_every": 3,
},
)
obj.set_gguf_parameters()
obj.gguf_writer.add_context_length.assert_called_with(1048576)
obj.gguf_writer.add_embedding_length.assert_called_with(4096)
obj.gguf_writer.add_block_count.assert_called()
obj.gguf_writer.add_head_count.assert_called_with(0) # RWKV-specific
obj.gguf_writer.add_wkv_head_size.assert_called_with(64)
def test_rwkv7_calc_lora_rank(self):
"""Test Rwkv7Model.calc_lora_rank computes correctly."""
from auto_round.export.export_to_gguf.conversion.rwkv import Rwkv7Model
obj = _make_mock_model(
Rwkv7Model,
{
"num_hidden_layers": 2,
"head_dim": 64,
"hidden_size": 4096,
"intermediate_size": 28672,
"norm_eps": 1e-5,
},
)
# calc_lora_rank = max(1, round(hidden_size ** exponent * multiplier / 32)) * 32
result = obj.calc_lora_rank(4096, 0.5, 1.8)
# 4096 ** 0.5 = 64, 64 * 1.8 = 115.2, /32 = 3.6, round = 4, *32 = 128
assert result == 128
def test_rwkv7_filter_tensors_unifies_names(self):
"""Test Rwkv7Model.filter_tensors unifies tensor name patterns."""
from auto_round.export.export_to_gguf.conversion.rwkv import Rwkv7Model
obj = _make_mock_model(
Rwkv7Model,
{
"num_hidden_layers": 2,
"head_dim": 64,
"hidden_size": 4096,
"intermediate_size": 28672,
"norm_eps": 1e-5,
},
)
# "blocks" -> "layers", "ffn" -> "feed_forward"
name, gen = "model.blocks.0.ffn.linear.weight", lambda: None
result = obj.filter_tensors((name, gen))
assert result is not None
assert "layers" in result[0]
assert "feed_forward" in result[0]
def test_rwkv7_filter_tensors_self_attn_renaming(self):
"""Test self_attn and attn are renamed to attention."""
from auto_round.export.export_to_gguf.conversion.rwkv import Rwkv7Model
obj = _make_mock_model(
Rwkv7Model,
{
"num_hidden_layers": 2,
"head_dim": 64,
"hidden_size": 4096,
"intermediate_size": 28672,
"norm_eps": 1e-5,
},
)
name, gen = "model.layers.0.self_attn.q_proj.weight", lambda: None
result = obj.filter_tensors((name, gen))
assert result is not None
assert "attention" in result[0]
# ==============================================================================
# smallthinker.py tests
# ==============================================================================
class TestSmallThinkerConversion:
"""Tests for SmallThinker conversion module."""
def test_smallthinker_prepare_tensors_unprocessed_experts_error(self):
"""Test unprocessed experts raise ValueError."""
from auto_round.export.export_to_gguf.conversion.smallthinker import SmallThinkerModel
obj = _make_mock_model(
SmallThinkerModel,
{
"num_hidden_layers": 2,
"hidden_size": 2048,
"moe_num_primary_experts": 8,
},
)
obj._experts = [{"unprocessed.tensor": None}]
obj.tensor_map.mapping = {"tensor": ("KEY", "tensor_name")}
with (
patch("auto_round.export.export_to_gguf.conversion.base.ModelBase.prepare_tensors"),
pytest.raises(ValueError, match="Unprocessed experts"),
):
obj.prepare_tensors()
def test_smallthinker_set_gguf_parameters_expert_gating(self):
"""Test expert gating function is set correctly."""
from auto_round.export.export_to_gguf.conversion.smallthinker import SmallThinkerModel
obj = _make_mock_model(
SmallThinkerModel,
{
"num_hidden_layers": 2,
"hidden_size": 2048,
"moe_num_primary_experts": 8,
"moe_num_active_primary_experts": 2,
"moe_ffn_hidden_size": 8192,
"moe_primary_router_apply_softmax": True,
},
)
obj.set_gguf_parameters()
obj.gguf_writer.add_expert_gating_func.assert_called()
from auto_round.export.export_to_gguf.conversion.smallthinker import SmallThinkerModel
obj = _make_mock_model(
SmallThinkerModel,
{
"moe_num_primary_experts": 32,
"moe_num_active_primary_experts": 4,
"moe_ffn_hidden_size": 4096,
"moe_primary_router_apply_softmax": True,
"max_position_embeddings": 32768,
"hidden_size": 4096,
"intermediate_size": 16384,
"num_attention_heads": 32,
},
)
with patch.object(SmallThinkerModel.__mro__[1], "set_gguf_parameters", lambda self: None):
obj.set_gguf_parameters()
from auto_round.export.export_to_gguf.conversion.base import gguf
w = obj.gguf_writer
w.add_expert_count.assert_called_once_with(32)
w.add_expert_used_count.assert_called_once_with(4)
w.add_expert_feed_forward_length.assert_called_once_with(4096)
w.add_expert_gating_func.assert_called_once_with(gguf.ExpertGatingFuncType.SOFTMAX)
def test_set_gguf_parameters_with_sigmoid(self):
"""Test SmallThinkerModel.set_gguf_parameters writes SIGMOID by default."""
from auto_round.export.export_to_gguf.conversion.smallthinker import SmallThinkerModel
obj = _make_mock_model(
SmallThinkerModel,
{
"moe_num_primary_experts": 32,
"moe_num_active_primary_experts": 4,
"moe_ffn_hidden_size": 4096,
"moe_primary_router_apply_softmax": False,
"max_position_embeddings": 32768,
"hidden_size": 4096,
"intermediate_size": 16384,
"num_attention_heads": 32,
},
)
with patch.object(SmallThinkerModel.__mro__[1], "set_gguf_parameters", lambda self: None):
obj.set_gguf_parameters()
from auto_round.export.export_to_gguf.conversion.base import gguf
obj.gguf_writer.add_expert_gating_func.assert_called_once_with(gguf.ExpertGatingFuncType.SIGMOID)
# ==============================================================================
# step3.py tests
# ==============================================================================
class TestStep3Conversion:
"""Tests for Step3 conversion module."""
def test_step35_filter_tensors_router_bias(self):
"""Test router_bias tensor gets .bias suffix appended."""
from auto_round.export.export_to_gguf.conversion.step3 import Step35Model
obj = _make_mock_model(
Step35Model,
{
"num_hidden_layers": 2,
"hidden_size": 4096,
"num_attention_heads": 32,
"num_attention_groups": 32,
"head_dim": 128,
"sliding_window": 4096,
"moe_num_experts": 8,
"moe_top_k": 2,
"moe_intermediate_size": 14336,
"share_expert_dim": 2048,
"rms_norm_eps": 1e-5,
"layer_types": ["full_attention"] * 2,
"partial_rotary_factors": [1.0] * 2,
},
)
# Step35Model.index_tensors() normally stashes the trunk layer count into
# this class attribute before filter_tensors() runs; the mock bypasses
# __init__/index_tensors, so set it explicitly here.
Step35Model._n_main_layers = 2
name, gen = "model.layers.0.moe.router_bias", lambda: None
result = obj.filter_tensors((name, gen))
assert result is not None
assert result[0] == "model.layers.0.moe.router_bias.bias"
def test_step35_modify_tensors_norm_plus_one(self):
"""Test norm.weight gets +1 added."""
from auto_round.export.export_to_gguf.conversion.step3 import Step35Model
obj = _make_mock_model(
Step35Model,
{
"num_hidden_layers": 2,
"hidden_size": 4096,
"num_attention_heads": 32,
"num_attention_groups": 32,
"head_dim": 128,
"sliding_window": 4096,
"moe_num_experts": 8,
"moe_top_k": 2,
"moe_intermediate_size": 14336,
"share_expert_dim": 2048,
"rms_norm_eps": 1e-5,
"layer_types": ["full_attention"] * 2,
"partial_rotary_factors": [1.0] * 2,
},
)
obj.rope_parameters = {}
data = torch.ones(4096) * 0.5
results = list(obj.modify_tensors(data, "model.layers.0.input_layernorm.weight", bid=0))
assert torch.allclose(results[0][1], torch.ones(4096) * 1.5)
def test_step35_generate_extra_tensors_llama3_rope(self):
"""Test generate_extra_tensors yields rope freqs for llama3 rope scaling."""
from auto_round.export.export_to_gguf.conversion.step3 import Step35Model
obj = _make_mock_model(
Step35Model,
{
"num_hidden_layers": 2,
"hidden_size": 4096,
"num_attention_heads": 32,
"num_attention_groups": 32,
"head_dim": 128,
"sliding_window": 4096,
"moe_num_experts": 8,
"moe_top_k": 2,
"moe_intermediate_size": 14336,
"share_expert_dim": 2048,
"rms_norm_eps": 1e-5,
"layer_types": ["full_attention"] * 2,
"partial_rotary_factors": [1.0] * 2,
"rope_theta": 10000.0,
},
)
obj.rope_parameters = {
"rope_type": "llama3",
"factor": 8.0,
"low_freq_factor": 1.0,
"high_freq_factor": 4.0,
"original_max_position_embeddings": 8192,
}
results = list(obj.generate_extra_tensors())
assert len(results) == 1
# The tensor name contains ROPE_FREQS
assert "ROPE_FREQS" in results[0][0]
assert results[0][1].dtype == torch.float32
def test_step35_generate_extra_tensors_non_llama3_returns_empty(self):
"""Test generate_extra_tensors returns empty for non-llama3 rope."""
from auto_round.export.export_to_gguf.conversion.step3 import Step35Model
obj = _make_mock_model(
Step35Model,
{
"num_hidden_layers": 2,
"hidden_size": 4096,
"num_attention_heads": 32,
"num_attention_groups": 32,
"head_dim": 128,
"sliding_window": 4096,
"moe_num_experts": 8,
"moe_top_k": 2,
"moe_intermediate_size": 14336,
"share_expert_dim": 2048,
"rms_norm_eps": 1e-5,
"layer_types": ["full_attention"] * 2,
"partial_rotary_factors": [1.0] * 2,
"rope_theta": 10000.0,
},
)
obj.rope_parameters = {
"rope_type": "linear",
"factor": 2.0,
}
results = list(obj.generate_extra_tensors())
assert len(results) == 0
# ==============================================================================
# t5.py tests
# ==============================================================================
class TestT5Conversion:
"""Tests for T5 conversion module."""
def test_modify_tensors_shared_token_first_occurrence(self):
"""Test first shared token embedding is used."""
from auto_round.export.export_to_gguf.conversion.t5 import T5Model
obj = _make_mock_model(
T5Model,
{
"num_decoder_layers": 2,
"d_model": 512,
"d_ff": 2048,
"num_heads": 8,
"d_kv": 64,
"layer_norm_epsilon": 1e-6,
"relative_attention_num_buckets": 32,
"decoder_start_token_id": 0,
},
)
obj.shared_token_embeddings_found = False
data = torch.randn(32000, 512)
results = list(obj.modify_tensors(data, "decoder.embed_tokens.weight", bid=None))
assert len(results) == 1
assert results[0][0] == "shared.weight"
assert obj.shared_token_embeddings_found is True
def test_modify_tensors_shared_token_second_occurrence_skipped(self):
"""Test second shared token embedding is skipped."""
from auto_round.export.export_to_gguf.conversion.t5 import T5Model
obj = _make_mock_model(
T5Model,
{
"num_decoder_layers": 2,
"d_model": 512,
"d_ff": 2048,
"num_heads": 8,
"d_kv": 64,
"layer_norm_epsilon": 1e-6,
"relative_attention_num_buckets": 32,
"decoder_start_token_id": 0,
},
)
obj.shared_token_embeddings_found = True # Already found
data = torch.randn(32000, 512)
results = list(obj.modify_tensors(data, "encoder.embed_tokens.weight", bid=None))
assert len(results) == 0
# ==============================================================================
# ultravox.py tests
# ==============================================================================
class TestUltravoxConversion:
"""Tests for Ultravox conversion module."""
def test_glmasr_filter_tensors_audio_encoder(self):
"""Test audio_encoder.* tensors are renamed correctly."""
from auto_round.export.export_to_gguf.conversion.ultravox import GlmASRWhisperEncoderModel
obj = _make_mock_model(
GlmASRWhisperEncoderModel,
{
"hidden_size": 1024,
"intermediate_size": 4096,
"num_attention_heads": 16,
"num_mel_bins": 80,
"d_model": 1024,
"encoder_ffn_dim": 4096,
"encoder_attention_heads": 16,
"merge_factor": 4,
},
)
obj.global_config = {"merge_factor": 4}
obj.hparams_vision = {}
# Whisper prefix should be stripped
name, gen = "audio_encoder.whisper.layer1.weight", lambda: None
result = obj.filter_tensors((name, gen))
assert result is not None
assert "audio_tower." in result[0]
def test_glmasr_filter_tensors_skips_lm_tensors(self):
"""Test model.* and lm_head.* tensors are skipped."""
from auto_round.export.export_to_gguf.conversion.ultravox import GlmASRWhisperEncoderModel
obj = _make_mock_model(
GlmASRWhisperEncoderModel,
{
"hidden_size": 1024,
"intermediate_size": 4096,
"num_attention_heads": 16,
"num_mel_bins": 80,
"d_model": 1024,
"encoder_ffn_dim": 4096,
"encoder_attention_heads": 16,
"merge_factor": 4,
},
)
obj.global_config = {"merge_factor": 4}
obj.hparams_vision = {}
result = obj.filter_tensors(("model.layers.0.weight", lambda: None))
assert result is None
result = obj.filter_tensors(("lm_head.weight", lambda: None))
assert result is None
def test_whisper_encoder_tensor_force_quant_conv(self):
"""Test conv weights are forced to F16."""
from auto_round.export.export_to_gguf.conversion.ultravox import WhisperEncoderModel
obj = _make_mock_model(
WhisperEncoderModel,
{
"hidden_size": 1024,
"intermediate_size": 4096,
"num_attention_heads": 16,
"num_mel_bins": 80,
"d_model": 1024,
"encoder_ffn_dim": 4096,
"encoder_attention_heads": 16,
},
)
obj.hparams_vision = {}
result = obj.tensor_force_quant("audio.conv1.weight", "audio.conv1.weight", 0, 4)
from auto_round.export.export_to_gguf.conversion.base import gguf
assert result == gguf.GGMLQuantizationType.F16
# ==============================================================================
# refact.py tests
# ==============================================================================
class TestRefactConversion:
"""Tests for Refact conversion module."""
def test_modify_tensors_attn_q(self):
"""Test attn.q.weight is passed through."""
from auto_round.export.export_to_gguf.conversion.refact import RefactModel
obj = _make_mock_model(
RefactModel,
{
"num_hidden_layers": 2,
"n_embd": 2048,
"n_positions": 2048,
"n_head": 16,
"layer_norm_epsilon": 1e-5,
},
)
data = torch.randn(2048, 2048)
results = list(obj.modify_tensors(data, "transformer.h.0.attn.q.weight", bid=0))
assert len(results) == 1
def test_modify_tensors_attn_kv(self):
"""Test attn.kv.weight is split into k and v."""
from auto_round.export.export_to_gguf.conversion.refact import RefactModel
obj = _make_mock_model(
RefactModel,
{
"num_hidden_layers": 2,
"n_embd": 2048,
"n_positions": 2048,
"n_head": 16,
"layer_norm_epsilon": 1e-5,
},
)
# k and v each have 1 head_dim = n_embd/n_head = 128
head_dim = 128
data = torch.randn(head_dim * 2, 2048) # [k_head + v_head, hidden]
results = list(obj.modify_tensors(data, "transformer.h.0.attn.kv.weight", bid=0))
# Should yield 2 results: ATTN_K and ATTN_V
assert len(results) == 2
assert "ATTN_K" in results[0][0]
assert "ATTN_V" in results[1][0]
def test_modify_tensors_gate_up_proj(self):
"""Test gate_up_proj.weight is split into gate and up."""
from auto_round.export.export_to_gguf.conversion.refact import RefactModel
obj = _make_mock_model(
RefactModel,
{
"num_hidden_layers": 2,
"n_embd": 2048,
"n_positions": 2048,
"n_head": 16,
"layer_norm_epsilon": 1e-5,
},
)
hidden_dim = 2048
inner_dim = 4 * hidden_dim
hidden_dim_calc = int(2 * inner_dim / 3)
multiple_of = 256
ff_dim = multiple_of * ((hidden_dim_calc + multiple_of - 1) // multiple_of) # 2730
# gate_up_proj is [ff_dim*2, hidden] = [5460, 2048]
data = torch.randn(ff_dim * 2, hidden_dim)
results = list(obj.modify_tensors(data, "transformer.h.0.mlp.gate_up_proj.weight", bid=0))
# Should yield 2 results: FFN_GATE and FFN_UP
assert len(results) == 2
# ==============================================================================
# wavtokenizer.py tests
# ==============================================================================
class TestWavTokenizerConversion:
"""Tests for WavTokenizer conversion module."""
def test_filter_tensors_skips_codebook_tensors(self):
"""Test codebook.* tensors are skipped."""
from auto_round.export.export_to_gguf.conversion.wavtokenizer import WavTokenizerDecModel
obj = _make_mock_model(
WavTokenizerDecModel,
{
"vocab_size": 1024,
"n_embd_features": 128,
"n_ff": 2048,
"group_norm_epsilon": 1e-5,
"group_norm_groups": 32,
"posnet": {"n_embd": 512, "n_layer": 4},
"convnext": {"n_embd": 256, "n_layer": 3},
},
)
# Should skip codebook tensors
for suffix in ["codebook.cluster_size", "codebook.embed_avg", "codebook.inited"]:
result = obj.filter_tensors((f"model.{suffix}", lambda: None))
assert result is None
def test_filter_tensors_keeps_other_tensors(self):
"""Test non-codebook tensors are kept."""
from auto_round.export.export_to_gguf.conversion.wavtokenizer import WavTokenizerDecModel
obj = _make_mock_model(
WavTokenizerDecModel,
{
"vocab_size": 1024,
"n_embd_features": 128,
"n_ff": 2048,
"group_norm_epsilon": 1e-5,
"group_norm_groups": 32,
"posnet": {"n_embd": 512, "n_layer": 4},
"convnext": {"n_embd": 256, "n_layer": 3},
},
)
result = obj.filter_tensors(("model.encoder.conv.weight", lambda: None))
assert result is not None
# ==============================================================================
# baichuan.py tests
# ==============================================================================
class TestBaichuanConversion:
"""Tests for Baichuan conversion module."""
def test_reverse_hf_permute_standard(self):
"""Test _reverse_hf_permute with n_head == n_kv_head (no GQA)."""
from auto_round.export.export_to_gguf.conversion.baichuan import BaichuanModel
obj = _make_mock_model(BaichuanModel)
# n_head=8, n_kv_head defaults to None -> equals n_head
n_head = 8
n_embd = 64
weights = torch.randn(n_head * n_embd, 128)
result = obj._reverse_hf_permute(weights, n_head)
assert result.shape == weights.shape
def test_reverse_hf_permute_gqa(self):
"""Test _reverse_hf_permute with grouped-query attention (n_head != n_kv_head)."""
from auto_round.export.export_to_gguf.conversion.baichuan import BaichuanModel
obj = _make_mock_model(BaichuanModel)
# GQA: 8 q heads, 2 kv heads -> n_kv_head=2 enters the GQA branch
n_head, n_kv_head = 8, 2
n_embd = 64
weights = torch.randn(n_head * n_embd, 128)
result = obj._reverse_hf_permute(weights, n_head, n_kv_head)
assert result.shape == weights.shape
def test_reverse_hf_permute_part(self):
"""Test _reverse_hf_permute_part splits the W_pack weight into q/k parts."""
from auto_round.export.export_to_gguf.conversion.baichuan import BaichuanModel
obj = _make_mock_model(BaichuanModel)
# W_pack has 3 * n_embd rows (Q, K, V packed)
n_head = 8
n_head_kv = 2
n_embd = 64
weights = torch.randn(3 * n_embd, 128)
# n_part=0 -> Q section
q_part = obj._reverse_hf_permute_part(weights, 0, n_head)
assert q_part.shape == (n_embd, 128)
# n_part=1 -> K section (uses n_head_kv)
k_part = obj._reverse_hf_permute_part(weights, 1, n_head, n_head_kv)
assert k_part.shape == (n_embd, 128)
def test_reverse_hf_part(self):
"""Test _reverse_hf_part splits the W_pack weight into v part (no permute)."""
from auto_round.export.export_to_gguf.conversion.baichuan import BaichuanModel
obj = _make_mock_model(BaichuanModel)
n_embd = 64
weights = torch.randn(3 * n_embd, 128)
# n_part=2 -> V section, no permutation
v_part = obj._reverse_hf_part(weights, 2)
assert v_part.shape == (n_embd, 128)
assert torch.equal(v_part, weights[2 * n_embd :, :])
def test_set_vocab_delegates_to_sentencepiece(self):
"""Test set_vocab calls the sentencepiece vocab setter."""
from auto_round.export.export_to_gguf.conversion.baichuan import BaichuanModel
obj = _make_mock_model(BaichuanModel)
with patch.object(obj, "_set_vocab_sentencepiece") as mock_spm:
obj.set_vocab()
mock_spm.assert_called_once_with()
def test_set_gguf_parameters(self):
"""Test set_gguf_parameters writes gguf metadata + baichuan-specific fields."""
from auto_round.export.export_to_gguf.conversion.baichuan import BaichuanModel
obj = _make_mock_model(
BaichuanModel,
{
"hidden_size": 512,
"num_attention_heads": 8,
"num_key_value_heads": 8,
"max_position_embeddings": 2048,
"intermediate_size": 2048,
},
)
with patch.object(BaichuanModel.__mro__[1], "set_gguf_parameters", lambda self: None):
obj.set_gguf_parameters()
obj.gguf_writer.add_tensor_data_layout.assert_called_once_with("Meta AI original pth")
obj.gguf_writer.add_rope_dimension_count.assert_called_once_with(512 // 8)
def test_modify_tensors_w_pack_unpack(self):
"""Test modify_tensors unpacks W_pack into Q, K, V at the right block id."""
from auto_round.export.export_to_gguf.conversion.baichuan import BaichuanModel
obj = _make_mock_model(
BaichuanModel,
{
"num_attention_heads": 8,
"num_key_value_heads": 2,
},
)
# Simulate W_pack weight of shape (3*n_embd, n_embd)
n_embd = 64
bid = 5
w_pack = torch.randn(3 * n_embd, n_embd)
results = list(obj.modify_tensors(w_pack, f"model.layers.{bid}.self_attn.W_pack.weight", bid))
# Should yield 3 tensors: Q, K, V
assert len(results) == 3
for name, tensor in results:
assert tensor.shape == (n_embd, n_embd)
def test_modify_tensors_non_w_pack_delegates(self):
"""Test modify_tensors for non-W_pack tensors delegates to parent mapping."""
from auto_round.export.export_to_gguf.conversion.baichuan import BaichuanModel
obj = _make_mock_model(
BaichuanModel,
{
"num_attention_heads": 8,
"num_key_value_heads": 8,
},
)
# Mock parent modify_tensors to return a known list. Use a simple function
# instead of MagicMock so it doesn't interfere with generator exhaustion.
n_embd = 64
fake_data = torch.randn(n_embd, n_embd)
# Patch the parent class's modify_tensors via the MRO chain.
# BaichuanModel.__mro__[1] is TextModel which inherits modify_tensors from ModelBase.
def parent_modify(self, data_torch, name, bid):
yield ("mapped.weight", data_torch)
# Patch the bound self.modify_tensors call on the instance directly so that
# the recursive self.modify_tensors(...) in the else branch hits our stub.
obj.modify_tensors = lambda d, n, b: parent_modify(obj, d, n, b)
obj.map_tensor_name = lambda name, try_suffixes=(".weight", ".bias"): name
# Pass bid=None so we hit the else branch
results = list(BaichuanModel.modify_tensors(obj, fake_data, "model.embed_tokens.weight", None))
assert results == [("mapped.weight", fake_data)]
# ==============================================================================
# maincoder.py tests
# ==============================================================================
class TestMaincoderConversion:
"""Tests for Maincoder conversion module."""
def test_set_gguf_parameters_with_head_dim(self):
"""Test set_gguf_parameters writes rope dimension when head_dim is set."""
from auto_round.export.export_to_gguf.conversion.maincoder import MaincoderModel
obj = _make_mock_model(
MaincoderModel,
{
"head_dim": 128,
"max_position_embeddings": 2048,
"hidden_size": 512,
"intermediate_size": 2048,
"num_attention_heads": 8,
},
)
with patch.object(MaincoderModel.__mro__[1], "set_gguf_parameters", lambda self: None):
obj.set_gguf_parameters()
obj.gguf_writer.add_rope_dimension_count.assert_called_once_with(128)
# ==============================================================================
# codeshell.py tests
# ==============================================================================
class TestCodeShellConversion:
"""Tests for CodeShell conversion module."""
def test_set_gguf_parameters(self):
"""Test set_gguf_parameters writes all CodeShell gguf fields."""
from auto_round.export.export_to_gguf.conversion.codeshell import CodeShellModel
obj = _make_mock_model(
CodeShellModel,
{
"n_positions": 8192,
"n_embd": 2048,
"n_head": 16,
"num_query_groups": 1,
"layer_norm_epsilon": 1e-5,
},
)
obj.set_gguf_parameters()
w = obj.gguf_writer
w.add_context_length.assert_called_once_with(8192)
w.add_embedding_length.assert_called_once_with(2048)
w.add_feed_forward_length.assert_called_once_with(4 * 2048)
w.add_head_count.assert_called_once_with(16)
w.add_head_count_kv.assert_called_once_with(1)
w.add_layer_norm_eps.assert_called_once_with(1e-5)
w.add_rope_freq_base.assert_called_once_with(10000.0)
from auto_round.export.export_to_gguf.conversion.base import gguf
w.add_rope_scaling_type.assert_called_once_with(gguf.RopeScalingType.LINEAR)
w.add_rope_scaling_factor.assert_called_once_with(1.0)
# ==============================================================================
# starcoder.py tests
# ==============================================================================
class TestStarCoderConversion:
"""Tests for StarCoder / StarCoder2 conversion modules."""
def test_starcoder_set_gguf_parameters(self):
"""Test StarCoderModel.set_gguf_parameters writes gguf metadata for starcoder."""
from auto_round.export.export_to_gguf.conversion.starcoder import StarCoderModel
obj = _make_mock_model(
StarCoderModel,
{
"n_positions": 8192,
"n_embd": 2048,
"n_head": 24,
"layer_norm_epsilon": 1e-5,
},
)
obj.set_gguf_parameters()
w = obj.gguf_writer
w.add_context_length.assert_called_once_with(8192)
w.add_embedding_length.assert_called_once_with(2048)
w.add_feed_forward_length.assert_called_once_with(4 * 2048)
w.add_head_count.assert_called_once_with(24)
# StarCoder is MQA (multi-query), always 1 kv head
w.add_head_count_kv.assert_called_once_with(1)
w.add_layer_norm_eps.assert_called_once_with(1e-5)
# ==============================================================================
# lighton_ocr.py tests
# ==============================================================================
class TestLightOnOCRConversion:
"""Tests for LightOnOCR conversion module."""
def test_set_gguf_parameters_writes_projector_type(self):
"""Test set_gguf_parameters writes LightOnOCR projector type."""
from auto_round.export.export_to_gguf.conversion.lighton_ocr import LightOnOCRVisionModel
obj = _make_mock_model(LightOnOCRVisionModel)
with patch.object(LightOnOCRVisionModel.__mro__[1], "set_gguf_parameters", lambda self: None):
obj.set_gguf_parameters()
from auto_round.export.export_to_gguf.conversion.base import gguf
obj.gguf_writer.add_clip_projector_type.assert_called_once_with(gguf.VisionProjectorType.LIGHTONOCR)
def test_filter_tensors_replaces_prefixes(self):
"""Test filter_tensors renames vision_encoder/vision_projection prefixes."""
from auto_round.export.export_to_gguf.conversion.lighton_ocr import LightOnOCRVisionModel
# Mock parent filter_tensors to just echo the renamed name back
def parent_filter(item):
name, gen = item
return (name, gen)
with patch.object(LightOnOCRVisionModel.__mro__[1], "filter_tensors", staticmethod(parent_filter)):
# Vision encoder path
result = LightOnOCRVisionModel.filter_tensors(("model.vision_encoder.layer.weight", lambda: None))
assert result[0] == "vision_tower.layer.weight"
# Vision projection path
result = LightOnOCRVisionModel.filter_tensors(("model.vision_projection.proj.weight", lambda: None))
assert result[0] == "multi_modal_projector.proj.weight"
# ==============================================================================
# dots1.py tests
# ==============================================================================
class TestDots1Conversion:
"""Tests for Dots1 conversion module."""
def test_set_gguf_parameters_writes_moe_metadata(self):
"""Test Dots1Model.set_gguf_parameters writes MoE-specific fields."""
from auto_round.export.export_to_gguf.conversion.dots1 import Dots1Model
obj = _make_mock_model(
Dots1Model,
{
"first_k_dense_replace": 3,
"n_shared_experts": 2,
"routed_scaling_factor": 1.5,
"norm_topk_prob": True,
"max_position_embeddings": 4096,
"hidden_size": 1024,
"intermediate_size": 4096,
"num_attention_heads": 16,
},
)
with patch.object(Dots1Model.__mro__[1], "set_gguf_parameters", lambda self: None):
obj.set_gguf_parameters()
w = obj.gguf_writer
w.add_leading_dense_block_count.assert_called_once_with(3)
w.add_expert_shared_count.assert_called_once_with(2)
w.add_expert_weights_scale.assert_called_once_with(1.5)
w.add_expert_weights_norm.assert_called_once_with(True)
# ==============================================================================
# sarashina2.py tests
# ==============================================================================
class TestSarashina2Conversion:
"""Tests for Sarashina2 conversion module."""
def test_text_filter_strips_llm_prefix(self):
"""Test Sarashina2VLTextModel.filter_tensors strips leading 'llm.' prefix."""
from auto_round.export.export_to_gguf.conversion.sarashina2 import Sarashina2VLTextModel
def parent_filter(item):
name, gen = item
return (name, gen)
with patch.object(Sarashina2VLTextModel.__mro__[1], "filter_tensors", staticmethod(parent_filter)):
result = Sarashina2VLTextModel.filter_tensors(("llm.layer.weight", lambda: None))
assert result[0] == "layer.weight"
def test_text_filter_drops_norm(self):
"""Test Sarashina2VLTextModel.filter_tensors returns None for 'norm.' prefix."""
from auto_round.export.export_to_gguf.conversion.sarashina2 import Sarashina2VLTextModel
result = Sarashina2VLTextModel.filter_tensors(("norm.weight", lambda: None))
assert result is None
# ==============================================================================
# cogvlm.py tests
# ==============================================================================
class TestCogVLMConversion:
"""Tests for CogVLM conversion module."""
def test_vision_filter_only_keeps_vision_prefix(self):
"""Test CogVLMVisionModel.filter_tensors only keeps tensors starting with 'model.vision.'."""
from auto_round.export.export_to_gguf.conversion.cogvlm import CogVLMVisionModel
# Non-vision tensor should be dropped
assert CogVLMVisionModel.filter_tensors(("model.embed_tokens.weight", lambda: None)) is None
# Vision tensor should pass through to parent
def parent_filter(item):
return item
with patch.object(CogVLMVisionModel.__mro__[1], "filter_tensors", staticmethod(parent_filter)):
result = CogVLMVisionModel.filter_tensors(("model.vision.layer.weight", lambda: None))
assert result[0] == "model.vision.layer.weight"
def test_vision_set_gguf_parameters(self):
"""Test CogVLMVisionModel.set_gguf_parameters writes projector type and layernorm eps."""
from auto_round.export.export_to_gguf.conversion.cogvlm import CogVLMVisionModel
obj = _make_mock_model(CogVLMVisionModel, {"layer_norm_eps": 1e-5})
with patch.object(CogVLMVisionModel.__mro__[1], "set_gguf_parameters", lambda self: None):
obj.set_gguf_parameters()
from auto_round.export.export_to_gguf.conversion.base import gguf
obj.gguf_writer.add_clip_projector_type.assert_called_once_with(gguf.VisionProjectorType.COGVLM)
obj.gguf_writer.add_vision_attention_layernorm_eps.assert_called_once_with(1e-5)
# ==============================================================================
# orion.py tests
# ==============================================================================
# ==============================================================================
# llama4.py tests
# ==============================================================================
class TestLlama4Conversion:
"""Tests for Llama4 conversion module."""
def test_set_gguf_parameters_asserts_gelu(self):
"""Test set_gguf_parameters asserts hidden_act is 'gelu' and writes use_gelu."""
from auto_round.export.export_to_gguf.conversion.llama4 import Llama4VisionModel
obj = _make_mock_model(
Llama4VisionModel,
{
"norm_eps": 1e-5,
"pixel_shuffle_ratio": 0.5,
"hidden_act": "gelu",
},
)
with patch.object(Llama4VisionModel.__mro__[1], "set_gguf_parameters", lambda self: None):
obj.set_gguf_parameters()
from auto_round.export.export_to_gguf.conversion.base import gguf
obj.gguf_writer.add_clip_projector_type.assert_called_once_with(gguf.VisionProjectorType.LLAMA4)
obj.gguf_writer.add_vision_attention_layernorm_eps.assert_called_once_with(1e-5)
obj.gguf_writer.add_vision_projector_scale_factor.assert_called_once_with(2)
obj.gguf_writer.add_vision_use_gelu.assert_called_once_with(True)
def test_filter_tensors_drops_non_vision(self):
"""Test filter_tensors returns None for non-vision tensors."""
from auto_round.export.export_to_gguf.conversion.llama4 import Llama4VisionModel
# Without multi_modal_projector or vision_model, drop the tensor
assert Llama4VisionModel.filter_tensors(("model.embed_tokens.weight", lambda: None)) is None
assert Llama4VisionModel.filter_tensors(("language_model.layer.weight", lambda: None)) is None
def test_filter_tensors_appends_weight_suffix(self):
"""Test filter_tensors adds '.weight' to positional_embedding_vlm tensors missing suffix."""
from auto_round.export.export_to_gguf.conversion.llama4 import Llama4VisionModel
def parent_filter(item):
return item
with patch.object(Llama4VisionModel.__mro__[1], "filter_tensors", staticmethod(parent_filter)):
result = Llama4VisionModel.filter_tensors(("vision_model.positional_embedding_vlm", lambda: None))
assert result[0] == "vision_model.positional_embedding_vlm.weight"
def test_modify_tensors_mmproj_linear_1(self):
"""Test Llama4VisionModel.modify_tensors maps multi_modal_projector.linear_1 to V_MMPROJ_FC."""
from auto_round.export.export_to_gguf.conversion.base import gguf
from auto_round.export.export_to_gguf.conversion.llama4 import Llama4VisionModel
obj = _make_mock_model(Llama4VisionModel)
data = torch.randn(8, 8)
result = list(obj.modify_tensors(data, "model.multi_modal_projector.linear_1.weight", bid=None))
# Should be renamed to V_MMPROJ_FC + '.weight'
assert result[0][0] == gguf.TENSOR_NAMES[gguf.MODEL_TENSOR.V_MMPROJ_FC] + ".weight"
def test_modify_tensors_passthrough(self):
"""Test Llama4VisionModel.modify_tensors passes through non-projector tensors."""
from auto_round.export.export_to_gguf.conversion.llama4 import Llama4VisionModel
obj = _make_mock_model(Llama4VisionModel)
with patch.object(Llama4VisionModel.__mro__[1], "modify_tensors", lambda self, d, n, b: iter([(n, d)])):
data = torch.randn(8, 8)
result = list(obj.modify_tensors(data, "vision_model.encoder.weight", bid=0))
assert torch.equal(result[0][1], data)
# ==============================================================================
# pixtral.py tests
# ==============================================================================
class TestPixtralConversion:
"""Tests for Pixtral conversion module."""
def test_set_gguf_parameters(self):
"""Test PixtralModel.set_gguf_parameters writes projector + vision params."""
from auto_round.export.export_to_gguf.conversion.pixtral import PixtralModel
obj = _make_mock_model(
PixtralModel,
{
"norm_eps": 1e-5,
"rope_theta": 10000.0,
"mm_projector_id": "patch_merge",
"spatial_merge_size": 2,
"hidden_size": 1024,
"intermediate_size": 4096,
"num_attention_heads": 16,
},
)
# find_vparam asserts hparams_vision is not None
obj.hparams_vision = {
"norm_eps": 1e-5,
"rope_theta": 10000.0,
"spatial_merge_size": 2,
"mm_projector_id": "patch_merge",
}
with patch.object(PixtralModel.__mro__[1], "set_gguf_parameters", lambda self: None):
obj.set_gguf_parameters()
from auto_round.export.export_to_gguf.conversion.base import gguf
obj.gguf_writer.add_clip_projector_type.assert_called_once_with(gguf.VisionProjectorType.PIXTRAL)
obj.gguf_writer.add_vision_use_silu.assert_called_once_with(True)
obj.gguf_writer.add_vision_spatial_merge_size.assert_called_once_with(2)
def test_map_tensor_name_mlp_adapter(self):
"""Test map_tensor_name maps the vision-language adapter weights to mm.*."""
from auto_round.export.export_to_gguf.conversion.pixtral import PixtralModel
obj = _make_mock_model(PixtralModel)
assert obj.map_tensor_name("vision_language_adapter.w_in.weight") == "mm.1.weight"
assert obj.map_tensor_name("vision_language_adapter.w_in.bias") == "mm.1.bias"
assert obj.map_tensor_name("vision_language_adapter.w_out.weight") == "mm.2.weight"
assert obj.map_tensor_name("vision_language_adapter.w_out.bias") == "mm.2.bias"
# ==============================================================================
# pangu.py tests
# ==============================================================================
# ==============================================================================
# bitnet.py tests
# ==============================================================================
class TestBitnetConversion:
"""Tests for Bitnet conversion module."""
def test_weight_quant_clamps_to_unit_range(self):
"""Test weight_quant output stays within [-1, +1] (BitNet ternary)."""
from auto_round.export.export_to_gguf.conversion.bitnet import BitnetModel
obj = _make_mock_model(BitnetModel)
# Continuous weights with extreme values
weight = torch.tensor([[0.5, -0.3, 2.0, -1.5], [0.1, -0.1, 1.0, -2.0]])
out = obj.weight_quant(weight)
# Values should be clamped within [-1, +1]
assert out.max() <= 1.0
assert out.min() >= -1.0
# Output dtype matches input dtype
assert out.dtype == weight.dtype
# Output shape matches input shape
assert out.shape == weight.shape
def test_set_gguf_parameters_writes_rope_scaling(self):
"""Test BitnetModel.set_gguf_parameters writes linear rope scaling."""
from auto_round.export.export_to_gguf.conversion.bitnet import BitnetModel
obj = _make_mock_model(
BitnetModel,
{
"max_position_embeddings": 4096,
"hidden_size": 2048,
"intermediate_size": 8192,
"num_attention_heads": 16,
},
)
with patch.object(BitnetModel.__mro__[1], "set_gguf_parameters", lambda self: None):
obj.set_gguf_parameters()
from auto_round.export.export_to_gguf.conversion.base import gguf
obj.gguf_writer.add_rope_scaling_type.assert_called_once_with(gguf.RopeScalingType.LINEAR)
obj.gguf_writer.add_rope_scaling_factor.assert_called_once_with(1.0)
# ==============================================================================
# mpt.py tests
# ==============================================================================
class TestMptConversion:
"""Tests for MPT conversion module."""
def test_set_gguf_parameters_with_kv_heads(self):
"""Test MPTModel.set_gguf_parameters writes kv heads when present."""
from auto_round.export.export_to_gguf.conversion.mpt import MPTModel
obj = _make_mock_model(
MPTModel,
{
"max_seq_len": 2048,
"d_model": 4096,
"n_heads": 32,
"attn_config": {"kv_n_heads": 8, "clip_qkv": None, "alibi": False},
},
)
obj.set_gguf_parameters()
w = obj.gguf_writer
w.add_context_length.assert_called_once_with(2048)
w.add_embedding_length.assert_called_once_with(4096)
w.add_feed_forward_length.assert_called_once_with(4 * 4096)
w.add_head_count.assert_called_once_with(32)
w.add_head_count_kv.assert_called_once_with(8)
w.add_layer_norm_eps.assert_called_once_with(1e-5)
w.add_max_alibi_bias.assert_called_once_with(0.0)
def test_set_gguf_parameters_with_alibi(self):
"""Test MPTModel.set_gguf_parameters writes alibi_bias_max when alibi=True."""
from auto_round.export.export_to_gguf.conversion.mpt import MPTModel
obj = _make_mock_model(
MPTModel,
{
"max_seq_len": 2048,
"d_model": 4096,
"n_heads": 32,
"attn_config": {"kv_n_heads": None, "clip_qkv": None, "alibi": True, "alibi_bias_max": 8.0},
},
)
obj.set_gguf_parameters()
obj.gguf_writer.add_max_alibi_bias.assert_called_once_with(8.0)
# ==============================================================================
# minimax.py tests
# ==============================================================================
class TestMinimaxConversion:
"""Tests for MiniMax M2 conversion module."""
def test_set_gguf_parameters(self):
"""Test MiniMaxM2Model.set_gguf_parameters writes expert + rope dims."""
from auto_round.export.export_to_gguf.conversion.minimax import MiniMaxM2Model
obj = _make_mock_model(
MiniMaxM2Model,
{
"intermediate_size": 8192,
"rotary_dim": 64,
"max_position_embeddings": 32768,
"hidden_size": 4096,
"num_attention_heads": 32,
},
)
with patch.object(MiniMaxM2Model.__mro__[1], "set_gguf_parameters", lambda self: None):
obj.set_gguf_parameters()
obj.gguf_writer.add_expert_feed_forward_length.assert_called_once_with(8192)
obj.gguf_writer.add_rope_dimension_count.assert_called_once_with(64)
# ==============================================================================
# falcon.py tests
# ==============================================================================
class TestFalconConversion:
"""Tests for Falcon conversion module."""
def test_set_gguf_parameters(self):
"""Test FalconModel.set_gguf_parameters uses jploski layout and writes metadata."""
from auto_round.export.export_to_gguf.conversion.falcon import FalconModel
obj = _make_mock_model(
FalconModel,
{
"num_attention_heads": 64,
"num_kv_heads": 8,
"hidden_size": 8192,
"layer_norm_epsilon": 1e-5,
},
)
obj.set_gguf_parameters()
w = obj.gguf_writer
w.add_tensor_data_layout.assert_called_once_with("jploski")
w.add_embedding_length.assert_called_once_with(8192)
w.add_feed_forward_length.assert_called_once_with(4 * 8192)
w.add_head_count.assert_called_once_with(64)
w.add_head_count_kv.assert_called_once_with(8)
# ==============================================================================
# gptneox.py tests
# ==============================================================================
class TestGptNeoxConversion:
"""Tests for GPTNeoX conversion module."""
def test_set_gguf_parameters(self):
"""Test GPTNeoXModel.set_gguf_parameters computes rope dim from rotary_pct."""
from auto_round.export.export_to_gguf.conversion.gptneox import GPTNeoXModel
obj = _make_mock_model(
GPTNeoXModel,
{
"max_position_embeddings": 2048,
"hidden_size": 6144,
"intermediate_size": 24576,
"rotary_pct": 0.25,
"num_attention_heads": 64,
"layer_norm_eps": 1e-5,
"use_parallel_residual": True,
},
)
obj.set_gguf_parameters()
w = obj.gguf_writer
# rotary_dim = int(0.25 * (6144 / 64)) = int(24) = 24
w.add_rope_dimension_count.assert_called_once_with(24)
w.add_parallel_residual.assert_called_once_with(True)
# ==============================================================================
# bloom.py tests
# ==============================================================================
class TestBloomConversion:
"""Tests for Bloom conversion module."""
def test_set_gguf_parameters(self):
"""Test BloomModel.set_gguf_parameters writes metadata with kv == q heads."""
from auto_round.export.export_to_gguf.conversion.bloom import BloomModel
obj = _make_mock_model(
BloomModel,
{
"hidden_size": 4096,
"n_head": 32,
"seq_length": 2048,
"layer_norm_epsilon": 1e-5,
},
)
obj.set_gguf_parameters()
w = obj.gguf_writer
w.add_embedding_length.assert_called_once_with(4096)
w.add_feed_forward_length.assert_called_once_with(4 * 4096)
w.add_head_count.assert_called_once_with(32)
w.add_head_count_kv.assert_called_once_with(32)
w.add_layer_norm_eps.assert_called_once_with(1e-5)
# ==============================================================================
# xverse.py tests
# ==============================================================================
class TestXverseConversion:
"""Tests for Xverse conversion module."""
def test_set_gguf_parameters(self):
"""Test XverseModel.set_gguf_parameters writes Meta layout and rope dim."""
from auto_round.export.export_to_gguf.conversion.xverse import XverseModel
obj = _make_mock_model(
XverseModel,
{
"hidden_size": 4096,
"num_attention_heads": 32,
"max_position_embeddings": 4096,
"intermediate_size": 16384,
},
)
with patch.object(XverseModel.__mro__[1], "set_gguf_parameters", lambda self: None):
obj.set_gguf_parameters()
obj.gguf_writer.add_tensor_data_layout.assert_called_once_with("Meta AI original pth")
obj.gguf_writer.add_rope_dimension_count.assert_called_once_with(4096 // 32)
def test_reverse_hf_permute(self):
"""Test _reverse_hf_permute round-trips shape."""
from auto_round.export.export_to_gguf.conversion.xverse import XverseModel
obj = _make_mock_model(XverseModel)
weights = torch.randn(4096, 1024)
out = obj._reverse_hf_permute(weights, n_head=32)
assert out.shape == weights.shape
def test_reverse_hf_permute_gqa(self):
"""Test _reverse_hf_permute with GQA (n_head != n_kv_head)."""
from auto_round.export.export_to_gguf.conversion.xverse import XverseModel
obj = _make_mock_model(XverseModel)
weights = torch.randn(4096, 1024)
out = obj._reverse_hf_permute(weights, n_head=32, n_kv_head=8)
assert out.shape == weights.shape
# ==============================================================================
# plm.py tests
# ==============================================================================
class TestPlmConversion:
"""Tests for PLM conversion module."""
def test_set_vocab_uses_gpt2(self):
"""Test PLMModel.set_vocab calls _set_vocab_gpt2."""
from auto_round.export.export_to_gguf.conversion.plm import PLMModel
obj = _make_mock_model(PLMModel)
with patch.object(obj, "_set_vocab_gpt2") as mock:
obj.set_vocab()
mock.assert_called_once_with()
def test_set_gguf_parameters_writes_kv_lora_params(self):
"""Test PLMModel.set_gguf_parameters writes kv_lora_rank and head dims."""
from auto_round.export.export_to_gguf.conversion.plm import PLMModel
obj = _make_mock_model(
PLMModel,
{
"vocab_size": 32000,
"kv_lora_rank": 64,
"qk_nope_head_dim": 32,
"qk_rope_head_dim": 16,
"v_head_dim": 64,
"max_position_embeddings": 4096,
"hidden_size": 2048,
"intermediate_size": 8192,
"num_attention_heads": 16,
},
)
with patch.object(PLMModel.__mro__[1], "set_gguf_parameters", lambda self: None):
obj.set_gguf_parameters()
w = obj.gguf_writer
w.add_vocab_size.assert_called_once_with(32000)
w.add_kv_lora_rank.assert_called_once_with(64)
# key_length = qk_nope_head_dim + qk_rope_head_dim = 32 + 16 = 48
w.add_key_length.assert_called_once_with(48)
w.add_value_length.assert_called_once_with(64)
w.add_rope_dimension_count.assert_called_once_with(16)
# ==============================================================================
# chameleon.py tests
# ==============================================================================
class TestChameleonConversion:
"""Tests for Chameleon conversion module."""
def test_set_gguf_parameters(self):
"""Test ChameleonModel.set_gguf_parameters writes swin_norm flag."""
from auto_round.export.export_to_gguf.conversion.chameleon import ChameleonModel
obj = _make_mock_model(
ChameleonModel,
{
"swin_norm": True,
"max_position_embeddings": 4096,
"hidden_size": 4096,
"intermediate_size": 16384,
"num_attention_heads": 32,
},
)
with patch.object(ChameleonModel.__mro__[1], "set_gguf_parameters", lambda self: None):
obj.set_gguf_parameters()
obj.gguf_writer.add_swin_norm.assert_called_once_with(True)
def test_set_vocab_delegates_to_gpt2(self):
"""Test set_vocab calls _set_vocab_gpt2."""
from auto_round.export.export_to_gguf.conversion.chameleon import ChameleonModel
obj = _make_mock_model(ChameleonModel)
with patch.object(obj, "_set_vocab_gpt2") as mock:
obj.set_vocab()
mock.assert_called_once_with()
def test_filter_tensors_drops_vqmodel(self):
"""Test filter_tensors returns None for model.vqmodel tensors."""
from auto_round.export.export_to_gguf.conversion.chameleon import ChameleonModel
result = ChameleonModel.filter_tensors(("model.vqmodel.encoder.weight", lambda: None))
assert result is None
# ==============================================================================
# dream.py tests
# ==============================================================================
class TestDreamConversion:
"""Tests for Dream conversion module."""
def test_set_gguf_parameters_disables_causal_attention(self):
"""Test DreamModel.set_gguf_parameters sets non-causal attention."""
from auto_round.export.export_to_gguf.conversion.dream import DreamModel
obj = _make_mock_model(
DreamModel,
{
"max_position_embeddings": 4096,
"hidden_size": 4096,
"intermediate_size": 16384,
"num_attention_heads": 32,
},
)
with patch.object(DreamModel.__mro__[1], "set_gguf_parameters", lambda self: None):
obj.set_gguf_parameters()
obj.gguf_writer.add_causal_attention.assert_called_once_with(False)
def test_set_gguf_parameters_with_mask_token_id(self):
"""Test DreamModel.set_gguf_parameters writes mask_token_id when present."""
from auto_round.export.export_to_gguf.conversion.dream import DreamModel
obj = _make_mock_model(
DreamModel,
{
"mask_token_id": 32000,
"max_position_embeddings": 4096,
"hidden_size": 4096,
"intermediate_size": 16384,
"num_attention_heads": 32,
},
)
with patch.object(DreamModel.__mro__[1], "set_gguf_parameters", lambda self: None):
obj.set_gguf_parameters()
obj.gguf_writer.add_mask_token_id.assert_called_once_with(32000)
# ==============================================================================
# dbrx.py tests
# ==============================================================================
class TestDbrxConversion:
"""Tests for Dbrx conversion module."""
def test_set_gguf_parameters(self):
"""Test DbrxModel.set_gguf_parameters writes MoE and rope params."""
from auto_round.export.export_to_gguf.conversion.dbrx import DbrxModel
obj = _make_mock_model(
DbrxModel,
{
"max_seq_len": 2048,
"d_model": 4096,
"n_heads": 32,
"ffn_config": {
"ffn_hidden_size": 14336,
"moe_num_experts": 16,
"moe_top_k": 4,
},
"attn_config": {
"kv_n_heads": 8,
"rope_theta": 10000.0,
"clip_qkv": 8.0,
},
},
)
obj.set_gguf_parameters()
w = obj.gguf_writer
w.add_context_length.assert_called_once_with(2048)
w.add_embedding_length.assert_called_once_with(4096)
w.add_feed_forward_length.assert_called_once_with(14336)
w.add_head_count.assert_called_once_with(32)
w.add_head_count_kv.assert_called_once_with(8)
w.add_rope_freq_base.assert_called_once_with(10000.0)
w.add_clamp_kqv.assert_called_once_with(8.0)
w.add_expert_count.assert_called_once_with(16)
w.add_expert_used_count.assert_called_once_with(4)
w.add_layer_norm_eps.assert_called_once_with(1e-5)
def test_tensor_force_quant_returns_n_dims_gt_1(self):
"""Test tensor_force_quant returns True when n_dims > 1, False otherwise."""
from auto_round.export.export_to_gguf.conversion.dbrx import DbrxModel
obj = _make_mock_model(DbrxModel)
assert obj.tensor_force_quant("name", "new_name", 0, n_dims=2) is True
assert obj.tensor_force_quant("name", "new_name", 0, n_dims=1) is False
# ==============================================================================
# gpt2.py tests
# ==============================================================================
class TestGpt2Conversion:
"""Tests for GPT2 conversion module."""
def test_gpt2_set_gguf_parameters(self):
"""Test GPT2Model.set_gguf_parameters writes GPT-2 metadata."""
from auto_round.export.export_to_gguf.conversion.gpt2 import GPT2Model
obj = _make_mock_model(
GPT2Model,
{
"n_ctx": 1024,
"n_embd": 768,
"n_head": 12,
"layer_norm_epsilon": 1e-5,
},
)
obj.set_gguf_parameters()
w = obj.gguf_writer
w.add_context_length.assert_called_once_with(1024)
w.add_embedding_length.assert_called_once_with(768)
w.add_feed_forward_length.assert_called_once_with(4 * 768)
w.add_head_count.assert_called_once_with(12)
w.add_layer_norm_eps.assert_called_once_with(1e-5)
# ==============================================================================
# afmoe.py tests
# ==============================================================================
class TestAfmoeConversion:
"""Tests for Afmoe conversion module."""
def test_filter_tensors_adds_bias_suffix(self):
"""Test filter_tensors appends '.bias' to expert_bias tensors."""
from auto_round.export.export_to_gguf.conversion.afmoe import AfmoeModel
def parent_filter(item):
return item
with patch.object(AfmoeModel.__mro__[1], "filter_tensors", staticmethod(parent_filter)):
result = AfmoeModel.filter_tensors(("layer.expert_bias", lambda: None))
assert result[0] == "layer.expert_bias.bias"
# ==============================================================================
# smallthinker.py tests
# ==============================================================================
# ==============================================================================
# openelm.py tests
# ==============================================================================
class TestOpenElmConversion:
"""Tests for OpenELM conversion module."""
def test_make_divisible_rounds_correctly(self):
"""Test _make_divisible rounds up to nearest multiple of divisor."""
from auto_round.export.export_to_gguf.conversion.openelm import OpenELMModel
# Standard rounding: int(v + divisor/2) // divisor * divisor
assert OpenELMModel._make_divisible(100, 64) == 128 # (100+32)//64*64 = 132//64*64 = 2*64
assert OpenELMModel._make_divisible(200, 64) == 192 # (200+32)//64*64 = 232//64*64 = 3*64
# 10% rule: never round down by more than 10%
# For v=80, divisor=64: new_v=64 (4*16), 64/80=0.8 < 0.9 -> +64 = 128
assert OpenELMModel._make_divisible(80, 64) == 128
# ==============================================================================
# command_r.py tests
# ==============================================================================
class TestCommandRConversion:
"""Tests for CommandR / Cohere2 conversion module."""
def test_command_r2_set_gguf_parameters(self):
"""Test CommandR2Model.set_gguf_parameters writes logit_scale + rope none."""
from auto_round.export.export_to_gguf.conversion.command_r import CommandR2Model
obj = _make_mock_model(
CommandR2Model,
{
"logit_scale": 0.0625,
"model_max_length": 131072,
"max_position_embeddings": 8192,
"hidden_size": 4096,
"intermediate_size": 16384,
"num_attention_heads": 32,
},
)
with patch.object(CommandR2Model.__mro__[1], "set_gguf_parameters", lambda self: None):
obj.set_gguf_parameters()
from auto_round.export.export_to_gguf.conversion.base import gguf
obj.gguf_writer.add_logit_scale.assert_called_once_with(0.0625)
obj.gguf_writer.add_rope_scaling_type.assert_called_once_with(gguf.RopeScalingType.NONE)
def test_cohere2_set_gguf_parameters(self):
"""Test Cohere2Model.set_gguf_parameters writes sliding_window and rope dims."""
from auto_round.export.export_to_gguf.conversion.command_r import Cohere2Model
obj = _make_mock_model(
Cohere2Model,
{
"logit_scale": 0.0625,
"sliding_window": 4096,
"vocab_size": 256000,
"rotary_pct": 0.25,
"hidden_size": 4096,
"num_attention_heads": 32,
"max_position_embeddings": 131072,
"intermediate_size": 16384,
},
)
with patch.object(Cohere2Model.__mro__[1], "set_gguf_parameters", lambda self: None):
obj.set_gguf_parameters()
from auto_round.export.export_to_gguf.conversion.base import gguf
w = obj.gguf_writer
w.add_logit_scale.assert_called_once_with(0.0625)
w.add_sliding_window.assert_called_once_with(4096)
w.add_vocab_size.assert_called_once_with(256000)
# rotary_dim = int(0.25 * (4096 / 32)) = int(32) = 32
w.add_rope_dimension_count.assert_called_once_with(32)
# ==============================================================================
# stablelm.py tests
# ==============================================================================
class TestStableLmConversion:
"""Tests for StableLM conversion module."""
def test_set_gguf_parameters(self):
"""Test StableLMModel.set_gguf_parameters computes rope dim from partial_rotary_factor."""
from auto_round.export.export_to_gguf.conversion.stablelm import StableLMModel
obj = _make_mock_model(
StableLMModel,
{
"max_position_embeddings": 4096,
"hidden_size": 4096,
"intermediate_size": 16384,
"partial_rotary_factor": 0.25,
"num_attention_heads": 32,
"num_key_value_heads": 8,
"use_parallel_residual": True,
"layer_norm_eps": 1e-5,
},
)
obj.set_gguf_parameters()
w = obj.gguf_writer
# rotary_dim = int(0.25 * (4096 / 32)) = int(32) = 32
w.add_rope_dimension_count.assert_called_once_with(32)
w.add_parallel_residual.assert_called_once_with(True)
w.add_layer_norm_eps.assert_called_once_with(1e-5)
# ==============================================================================
# mistral3.py tests
# ==============================================================================
class TestMistral3Conversion:
"""Tests for Mistral3 conversion module."""
def test_ministral3_set_gguf_parameters_with_yarn(self):
"""Test Ministral3Model.set_gguf_parameters writes yarn rope params."""
from auto_round.export.export_to_gguf.conversion.mistral3 import Mistral3Model
# The inner class
cls = Mistral3Model.Ministral3Model
obj = _make_mock_model(
cls,
{
"model_type": "ministral3",
"max_position_embeddings": 32768,
"hidden_size": 4096,
"intermediate_size": 16384,
"num_attention_heads": 32,
},
)
obj.rope_parameters = {
"rope_type": "yarn",
"mscale_all_dim": 1.0,
"llama_4_scaling_beta": 0.5,
}
with patch.object(cls.__mro__[1], "set_gguf_parameters", lambda self: None):
obj.set_gguf_parameters()
obj.gguf_writer.add_rope_scaling_yarn_log_mul.assert_called_once_with(1.0)
obj.gguf_writer.add_attn_temperature_scale.assert_called_once_with(0.5)
def test_ministral3_asserts_yarn_rope_type(self):
"""Test Ministral3Model.set_gguf_parameters asserts rope_type must be 'yarn'."""
from auto_round.export.export_to_gguf.conversion.mistral3 import Mistral3Model
cls = Mistral3Model.Ministral3Model
obj = _make_mock_model(
cls,
{
"model_type": "ministral3",
"max_position_embeddings": 32768,
"hidden_size": 4096,
"intermediate_size": 16384,
"num_attention_heads": 32,
},
)
obj.rope_parameters = {"rope_type": "linear", "mscale_all_dim": 1.0, "llama_4_scaling_beta": 0.5}
with (
patch.object(cls.__mro__[1], "set_gguf_parameters", lambda self: None),
pytest.raises(AssertionError, match="rope_type must be 'yarn'"),
):
obj.set_gguf_parameters()
# ==============================================================================
# internvl.py tests
# ==============================================================================
class TestInternVlConversion:
"""Tests for InternVL conversion module."""
def test_set_gguf_parameters_with_gelu(self):
"""Test InternVisionModel.set_gguf_parameters writes vision_use_gelu for gelu activation."""
from auto_round.export.export_to_gguf.conversion.internvl import InternVisionModel
obj = _make_mock_model(
InternVisionModel,
{
"layer_norm_eps": 1e-6,
"hidden_act": "gelu",
},
)
obj.hparams_vision = {
"image_size": 448,
"patch_size": 14,
"hidden_size": 3200,
"intermediate_size": 12800,
"num_attention_heads": 50,
"num_hidden_layers": 48,
}
obj.global_config = {"downsample_ratio": 0.5}
with patch.object(InternVisionModel.__mro__[1], "set_gguf_parameters", lambda self: None):
obj.set_gguf_parameters()
from auto_round.export.export_to_gguf.conversion.base import gguf
obj.gguf_writer.add_clip_projector_type.assert_called_once_with(gguf.VisionProjectorType.INTERNVL)
obj.gguf_writer.add_vision_use_gelu.assert_called_once_with(True)
obj.gguf_writer.add_vision_projector_scale_factor.assert_called_once_with(2)
# ==============================================================================
# smolvlm.py tests
# ==============================================================================
class TestSmolVlmConversion:
"""Tests for SmolVLM conversion module."""
def test_set_gguf_parameters_with_defaults(self):
"""Test SmolVLMModel.set_gguf_parameters writes IDEFICS3 projector and uses defaults."""
from auto_round.export.export_to_gguf.conversion.smolvlm import SmolVLMModel
obj = _make_mock_model(
SmolVLMModel,
{
"model_type": "smolvlm_vision",
"hidden_size": 1152,
"num_attention_heads": 16,
"intermediate_size": 3072,
"layer_norm_eps": 1e-5,
},
)
obj.hparams_vision = {
"image_size": 384,
"patch_size": 14,
"hidden_size": 1152,
"intermediate_size": 3072,
"num_attention_heads": 16,
"num_hidden_layers": 24,
}
obj.image_size = 384
obj.preprocessor_config = {"size": {"longest_edge": 384}}
obj.global_config = {"scale_factor": 2}
with patch.object(SmolVLMModel.__mro__[1], "set_gguf_parameters", lambda self: None):
obj.set_gguf_parameters()
from auto_round.export.export_to_gguf.conversion.base import gguf
obj.gguf_writer.add_clip_projector_type.assert_called_once_with(gguf.VisionProjectorType.IDEFICS3)
obj.gguf_writer.add_vision_use_gelu.assert_called_once_with(True)
obj.gguf_writer.add_vision_projector_scale_factor.assert_called_once_with(2)
obj.gguf_writer.add_vision_preproc_image_size.assert_called_once_with(384)
# ==============================================================================
# dotsocr.py tests
# ==============================================================================
class TestDotsOcrConversion:
"""Tests for DotsOCR conversion module."""
def test_set_gguf_parameters(self):
"""Test DotsOCRVisionModel.set_gguf_parameters writes DOTSOCR projector + preproc pixels."""
from auto_round.export.export_to_gguf.conversion.dotsocr import DotsOCRVisionModel
obj = _make_mock_model(DotsOCRVisionModel)
obj.hparams_vision = {
"image_size": 0,
"patch_size": 14,
"hidden_size": 1024,
"intermediate_size": 4096,
"num_attention_heads": 16,
"num_hidden_layers": 24,
"rms_norm_eps": 1e-5,
"spatial_merge_size": 2,
}
obj.preprocessor_config = {"min_pixels": 256, "max_pixels": 1280 * 28 * 28}
with patch.object(DotsOCRVisionModel.__mro__[1], "set_gguf_parameters", lambda self: None):
obj.set_gguf_parameters()
from auto_round.export.export_to_gguf.conversion.base import gguf
obj.gguf_writer.add_clip_projector_type.assert_called_once_with(gguf.VisionProjectorType.DOTSOCR)
obj.gguf_writer.add_vision_min_pixels.assert_called_once_with(256)
obj.gguf_writer.add_vision_max_pixels.assert_called_once_with(1280 * 28 * 28)
obj.gguf_writer.add_vision_projector_scale_factor.assert_called_once_with(2)
# ==============================================================================
# youtuvl.py tests
# ==============================================================================
class TestYoutuVlConversion:
"""Tests for YoutuVL conversion module."""
def test_set_gguf_parameters_with_gelu(self):
"""Test YoutuVLVisionModel.set_gguf_parameters writes YOUTUVL projector and use_gelu."""
from auto_round.export.export_to_gguf.conversion.youtuvl import YoutuVLVisionModel
obj = _make_mock_model(
YoutuVLVisionModel,
{
"layer_norm_eps": 1e-6,
"hidden_act": "gelu_pytorch_tanh",
"spatial_merge_size": 2,
"fullatt_block_indexes": [2, 5, 8, 11],
"window_size": 112,
},
)
obj.hparams_vision = {
"image_size": 560,
"patch_size": 14,
"hidden_size": 1280,
"intermediate_size": 5120,
"num_attention_heads": 16,
"num_hidden_layers": 24,
}
with patch.object(YoutuVLVisionModel.__mro__[1], "set_gguf_parameters", lambda self: None):
obj.set_gguf_parameters()
from auto_round.export.export_to_gguf.conversion.base import gguf
obj.gguf_writer.add_clip_projector_type.assert_called_once_with(gguf.VisionProjectorType.YOUTUVL)
obj.gguf_writer.add_vision_use_gelu.assert_called_once_with(True)
obj.gguf_writer.add_vision_spatial_merge_size.assert_called_once_with(2)
obj.gguf_writer.add_vision_window_size.assert_called_once_with(112)
obj.gguf_writer.add_vision_wa_layer_indexes.assert_called_once_with(layers=[2, 5, 8, 11])
# ==============================================================================
# chatglm.py tests
# ==============================================================================
class TestChatGlmConversion:
"""Tests for ChatGLM conversion module."""
def test_set_gguf_parameters_with_attention_dim(self):
"""Test ChatGLMModel.set_gguf_parameters uses attention_dim for rope when present."""
from auto_round.export.export_to_gguf.conversion.chatglm import ChatGLMModel
obj = _make_mock_model(
ChatGLMModel,
{
"hidden_size": 4096,
"num_attention_heads": 32,
"multi_query_group_num": 2,
"seq_length": 2048,
"ffn_hidden_size": 16384,
"layernorm_epsilon": 1e-5,
"attention_dim": 128,
"partial_rotary_factor": 0.5,
"rope_ratio": 1.0,
},
)
obj.set_gguf_parameters()
w = obj.gguf_writer
w.add_context_length.assert_called_once_with(2048)
w.add_embedding_length.assert_called_once_with(4096)
w.add_feed_forward_length.assert_called_once_with(16384)
w.add_head_count.assert_called_once_with(32)
w.add_head_count_kv.assert_called_once_with(2)
w.add_layer_norm_rms_eps.assert_called_once_with(1e-5)
# rope_dim = int(128 * 0.5) = 64
w.add_rope_dimension_count.assert_called_once_with(64)
w.add_add_bos_token.assert_called_once_with(False)
w.add_rope_freq_base.assert_called_once_with(10000.0)
def test_set_gguf_parameters_rope_ratio(self):
"""Test ChatGLMModel.set_gguf_parameters multiplies rope_freq by rope_ratio."""
from auto_round.export.export_to_gguf.conversion.chatglm import ChatGLMModel
obj = _make_mock_model(
ChatGLMModel,
{
"hidden_size": 4096,
"num_attention_heads": 32,
"ffn_hidden_size": 16384,
"layernorm_epsilon": 1e-5,
"rope_ratio": 2.0,
},
)
obj.set_gguf_parameters()
# rope_freq = 10000 * 2.0 = 20000
obj.gguf_writer.add_rope_freq_base.assert_called_once_with(20000.0)
def test_bpe_static_method(self):
"""Test ChatGLMModel.bpe static method merges bpe pairs greedily."""
from auto_round.export.export_to_gguf.conversion.chatglm import ChatGLMModel
# Simple rank: ab = 10, bc = 20, abc = 30
mergeable_ranks = {b"ab": 10, b"bc": 20, b"abc": 30}
result = ChatGLMModel.bpe(mergeable_ranks, b"abc")
# b"abc" should remain (since rank 30 wins for abc pair, no higher-merge path cheaper)
assert result == [b"abc"]
def test_bpe_with_smaller_rank(self):
"""Test ChatGLMModel.bpe splits token when a sub-merge has a smaller rank."""
from auto_round.export.export_to_gguf.conversion.chatglm import ChatGLMModel
# ab = 5 (smaller rank means more likely merge), cd = 10
mergeable_ranks = {b"ab": 5, b"cd": 10}
# Input: "abcd" -> [a, b, c, d] -> pairs: (a,b) rank 5 < (c,d) rank 10
# After merging (a,b), parts become [ab, c, d] with one pair (c, d) rank 10.
# (c, d) also gets merged, final result is [ab, cd].
result = ChatGLMModel.bpe(mergeable_ranks, b"abcd")
assert result == [b"ab", b"cd"]
# ==============================================================================
# jais.py tests
# ==============================================================================
class TestJaisConversion:
"""Tests for Jais / Jais2 conversion module."""
def test_jais2_set_gguf_parameters(self):
"""Test Jais2Model.set_gguf_parameters writes rope_dimension_count from head_dim."""
from auto_round.export.export_to_gguf.conversion.jais import Jais2Model
obj = _make_mock_model(
Jais2Model,
{
"head_dim": 128,
"max_position_embeddings": 8192,
"hidden_size": 4096,
"intermediate_size": 16384,
"num_attention_heads": 32,
},
)
with patch.object(Jais2Model.__mro__[1], "set_gguf_parameters", lambda self: None):
obj.set_gguf_parameters()
obj.gguf_writer.add_rope_dimension_count.assert_called_once_with(128)
def test_jais2_set_gguf_parameters_without_head_dim(self):
"""Test Jais2Model.set_gguf_parameters derives head_dim from hidden_size/num_heads."""
from auto_round.export.export_to_gguf.conversion.jais import Jais2Model
obj = _make_mock_model(
Jais2Model,
{
"hidden_size": 4096,
"num_attention_heads": 32,
"max_position_embeddings": 8192,
"intermediate_size": 16384,
},
)
with patch.object(Jais2Model.__mro__[1], "set_gguf_parameters", lambda self: None):
obj.set_gguf_parameters()
# 4096 / 32 = 128
obj.gguf_writer.add_rope_dimension_count.assert_called_once_with(128)
def test_jais_set_vocab_uses_gpt2(self):
"""Test JaisModel.set_vocab calls _set_vocab_gpt2."""
from auto_round.export.export_to_gguf.conversion.jais import JaisModel
obj = _make_mock_model(JaisModel)
with patch.object(obj, "_set_vocab_gpt2") as mock:
obj.set_vocab()
mock.assert_called_once_with()
def test_filter_tensors_drops_attn_bias(self):
"""Test JaisModel.filter_tensors returns None for .attn.bias tensors."""
from auto_round.export.export_to_gguf.conversion.jais import JaisModel
result = JaisModel.filter_tensors(("transformer.h.0.attn.bias", lambda: None))
assert result is None
def test_jais_modify_tensors_alibi_slopes(self):
"""Test JaisModel.modify_tensors computes max_alibi_bias from slopes."""
from auto_round.export.export_to_gguf.conversion.jais import JaisModel
obj = _make_mock_model(
JaisModel,
{
"n_head": 32,
"embeddings_scale": 1.0,
"width_scale": 1.0,
},
)
# Manually set max_alibi_bias (normally set in __init__)
obj.max_alibi_bias = 8.0
# max_alibi_bias starts at 8.0
assert obj.max_alibi_bias == 8.0
# Provide a slope tensor; first_val = 0.5 -> log2(0.5) = -1 -> -round(-1 * 32) = 32
# n_head_closest_log2 = 2**floor(log2(32)) = 32
data = torch.tensor([0.5] + [0.1] * 31)
with patch.object(JaisModel.__mro__[1], "modify_tensors", lambda self, d, n, b: iter([(n, d)])):
result = list(obj.modify_tensors(data, "transformer.h.0.attn.relative_pe.slopes", bid=0))
assert result == [] # function returns early
# max_alibi_bias should now be -round(-1 * 32) = 32
assert obj.max_alibi_bias == 32
def test_jais_modify_tensors_transpose(self):
"""Test JaisModel.modify_tensors transposes .c_attn/c_proj/c_fc/c_fc2 weights."""
from auto_round.export.export_to_gguf.conversion.jais import JaisModel
obj = _make_mock_model(
JaisModel,
{
"n_head": 32,
"embeddings_scale": 1.0,
"width_scale": 1.0,
},
)
data = torch.randn(8, 16)
with patch.object(JaisModel.__mro__[1], "modify_tensors", lambda self, d, n, b: iter([(n, d)])):
result = list(obj.modify_tensors(data, "transformer.h.0.attn.c_attn.weight", bid=0))
# After transpose(1, 0), shape is (16, 8)
assert result[0][1].shape == (16, 8)
# ==============================================================================
# grok.py tests
# ==============================================================================
class TestGrokConversion:
"""Tests for Grok conversion module."""
def test_set_gguf_parameters_with_yarn(self):
"""Test GrokModel.set_gguf_parameters writes yarn rope params."""
from auto_round.export.export_to_gguf.conversion.grok import GrokModel
obj = _make_mock_model(
GrokModel,
{
"head_dim": 128,
"hidden_size": 6144,
"num_attention_heads": 48,
"moe_intermediate_size": 2048,
"rope_type": "yarn",
"scaling_factor": 16.0,
"original_max_position_embeddings": 32768,
"extrapolation_factor": 1.0,
"attn_factor": 1.0,
"beta_fast": 32.0,
"beta_slow": 1.0,
"attn_temperature_len": 1024,
"attn_output_multiplier": 0.1,
"embedding_multiplier_scale": 0.5,
"output_multiplier_scale": 1.0,
"max_position_embeddings": 131072,
"intermediate_size": 24576,
},
)
with patch.object(GrokModel.__mro__[1], "set_gguf_parameters", lambda self: None):
obj.set_gguf_parameters()
from auto_round.export.export_to_gguf.conversion.base import gguf
w = obj.gguf_writer
w.add_attn_logit_softcapping.assert_called_once_with(30.0)
w.add_router_logit_softcapping.assert_called_once_with(30.0)
w.add_rope_scaling_type.assert_called_once_with(gguf.RopeScalingType.YARN)
w.add_rope_scaling_factor.assert_called_once_with(16.0)
w.add_attn_temperature_length.assert_called_once_with(1024)
w.add_attn_output_scale.assert_called_once_with(0.1)
w.add_embedding_scale.assert_called_once_with(0.5)
w.add_logit_scale.assert_called_once_with(1.0)
# ==============================================================================
# jamba.py tests
# ==============================================================================
class TestJambaConversion:
"""Tests for Jamba conversion module."""
def test_set_gguf_parameters(self):
"""Test JambaModel.set_gguf_parameters writes SSM and MoE fields."""
from auto_round.export.export_to_gguf.conversion.jamba import JambaModel
obj = _make_mock_model(
JambaModel,
{
"hidden_size": 4096,
"mamba_d_conv": 4,
"mamba_expand": 2,
"mamba_d_state": 16,
"mamba_dt_rank": 32,
"layer_norm_epsilon": 1e-6,
"num_key_value_heads": 8,
"attn_layer_offset": 1,
"attn_layer_period": 8,
"max_position_embeddings": 8192,
"intermediate_size": 14336,
"num_attention_heads": 32,
"num_local_experts": 16,
"num_experts_per_tok": 2,
},
)
obj.set_gguf_parameters()
w = obj.gguf_writer
w.add_block_count.assert_called_once_with(2)
w.add_context_length.assert_called_once_with(8192)
w.add_embedding_length.assert_called_once_with(4096)
# d_inner = mamba_expand * d_model = 2 * 4096 = 8192
w.add_ssm_inner_size.assert_called_once_with(8192)
w.add_ssm_state_size.assert_called_once_with(16)
w.add_ssm_conv_kernel.assert_called_once_with(4)
w.add_ssm_time_step_rank.assert_called_once_with(32)
w.add_expert_count.assert_called_once_with(16)
w.add_expert_used_count.assert_called_once_with(2)
def test_modify_tensors_a_log_negation(self):
"""Test JambaModel.modify_tensors negates exp of A_log."""
from auto_round.export.export_to_gguf.conversion.jamba import JambaModel
obj = _make_mock_model(
JambaModel,
{
"expert_layer_offset": 0,
"expert_layer_period": 100,
},
)
# A_log -> -exp(A_log)
data = torch.tensor([0.0, 1.0, 2.0])
result = list(obj.modify_tensors(data, "model.layers.0.mixer.A_log", bid=0))
# First value: -exp(0) = -1.0
# Second: -exp(1) ~= -2.7183
assert torch.allclose(result[0][1], torch.tensor([-1.0, -2.7183, -7.3891]), atol=1e-3)
def test_modify_tensors_ssm_conv1d_squeeze(self):
"""Test JambaModel.modify_tensors squeezes SSM_CONV1D tensors."""
from auto_round.export.export_to_gguf.conversion.jamba import JambaModel
obj = _make_mock_model(
JambaModel,
{
"expert_layer_offset": 0,
"expert_layer_period": 100,
},
)
# Without map_tensor_name mock, we can't easily hit SSM_CONV1D squeeze path,
# but we verify the function doesn't crash.
data = torch.randn(8)
result = list(obj.modify_tensors(data, "model.layers.0.ssm_conv1d.weight", bid=0))
# Should at least yield one tensor
assert len(result) >= 1
def test_modify_tensors_experts_merge_mini_jamba(self):
"""Test JambaModel.modify_tensors merges feed_forward.experts tensors (Mini-Jamba)."""
from auto_round.export.export_to_gguf.conversion.jamba import JambaModel
obj = _make_mock_model(
JambaModel,
{
"num_local_experts": 2,
"expert_layer_offset": 0,
"expert_layer_period": 1,
},
)
captured = []
obj.map_tensor_name = lambda n: n # identity mapping
obj.match_model_tensor_name = lambda *args, **kwargs: False
# 3 experts * 3 weights = 6 tensors trigger merge
for xid in range(2):
for wid in ["down_proj", "gate_proj", "up_proj"]:
ename = f"model.layers.0.moe.experts.{xid}.{wid}.weight"
captured.extend(list(obj.modify_tensors(torch.randn(8, 8), ename, bid=0)))
# Mini-Jamba renames .moe. -> .feed_forward.
# For bid=0 >= moe_offset=0 AND (0 - 0) % 1 == 0, so it's a MoE layer
# After rename, .experts.0. stays; merged_name becomes mlp.experts.{wid}.weight
assert any("mlp.experts.down_proj.weight" in n for n, _ in captured)
assert any("mlp.experts.gate_proj.weight" in n for n, _ in captured)
assert any("mlp.experts.up_proj.weight" in n for n, _ in captured)
# ==============================================================================
# januspro.py tests
# ==============================================================================
class TestJanusProConversion:
"""Tests for JanusPro conversion module."""
def test_text_filter_drops_vision_model(self):
"""Test JanusProModel.filter_tensors drops vision, aligner, generation tensors."""
from auto_round.export.export_to_gguf.conversion.januspro import JanusProModel
for skip in (
"model.vision_model.encoder",
"model.aligner.fc1",
"model.vqmodel.quantizer",
"model.generation_embeddings",
"model.generation_aligner",
"model.generation_head",
):
assert JanusProModel.filter_tensors((skip + ".weight", lambda: None)) is None
# ==============================================================================
# grovemoe.py tests
# ==============================================================================
class TestGroveMoeConversion:
"""Tests for GroveMoe conversion module."""
def test_set_gguf_parameters_writes_moe_defaults(self):
"""Test GroveMoeModel.set_gguf_parameters writes MoE + hardcoded per-group values."""
from auto_round.export.export_to_gguf.conversion.grovemoe import GroveMoeModel
obj = _make_mock_model(
GroveMoeModel,
{
"moe_intermediate_size": 2048,
"head_dim": 128,
"max_position_embeddings": 32768,
"hidden_size": 4096,
"intermediate_size": 12288,
"num_attention_heads": 32,
},
)
with patch.object(GroveMoeModel.__mro__[1], "set_gguf_parameters", lambda self: None):
obj.set_gguf_parameters()
w = obj.gguf_writer
w.add_expert_feed_forward_length.assert_called_once_with(2048)
w.add_expert_chunk_feed_forward_length.assert_called_once_with(128)
w.add_experts_per_group.assert_called_once_with(2)
w.add_expert_group_scale.assert_called_once_with(0.05)
def test_modify_tensors_drops_expert_bias(self):
"""Test GroveMoeModel.modify_tensors drops .expert_bias tensors."""
from auto_round.export.export_to_gguf.conversion.grovemoe import GroveMoeModel
obj = _make_mock_model(GroveMoeModel)
result = list(obj.modify_tensors(torch.zeros(8), "model.layers.0.mlp.experts.0.expert_bias", bid=0))
assert result == []
def test_modify_tensors_chunk_experts_merge(self):
"""Test GroveMoeModel.modify_tensors merges chunk_experts tensors."""
from auto_round.export.export_to_gguf.conversion.grovemoe import GroveMoeModel
obj = _make_mock_model(
GroveMoeModel,
{
"num_local_experts": 2,
},
)
captured = []
with patch.object(
GroveMoeModel.__mro__[1], "modify_tensors", lambda self, d, n, b: captured.append(n) or iter([(n, d)])
):
# n_experts // 2 = 1, so 3 tensors trigger merge
for xid in range(1):
for wid in ["down_proj", "gate_proj", "up_proj"]:
ename = f"model.layers.0.mlp.chunk_experts.{xid}.{wid}.weight"
list(obj.modify_tensors(torch.randn(8, 8), ename, bid=0))
# 3 merged names expected
assert any("chunk_experts.down_proj.weight" in n for n in captured)
assert any("chunk_experts.gate_proj.weight" in n for n in captured)
assert any("chunk_experts.up_proj.weight" in n for n in captured)
def test_modify_tensors_experts_merge(self):
"""Test GroveMoeModel.modify_tensors merges regular experts tensors."""
from auto_round.export.export_to_gguf.conversion.grovemoe import GroveMoeModel
obj = _make_mock_model(
GroveMoeModel,
{
"num_local_experts": 2,
},
)
captured = []
with patch.object(
GroveMoeModel.__mro__[1], "modify_tensors", lambda self, d, n, b: captured.append(n) or iter([(n, d)])
):
# 3 experts * 3 weights = 6 tensors trigger merge
for xid in range(2):
for wid in ["down_proj", "gate_proj", "up_proj"]:
ename = f"model.layers.0.mlp.experts.{xid}.{wid}.weight"
list(obj.modify_tensors(torch.randn(8, 8), ename, bid=0))
assert any("experts.down_proj.weight" in n for n in captured)
assert any("experts.gate_proj.weight" in n for n in captured)
assert any("experts.up_proj.weight" in n for n in captured)
# ==============================================================================
# falcon_h1.py tests
# ==============================================================================
class TestFalconH1Conversion:
"""Tests for FalconH1 conversion module."""
def test_set_gguf_parameters(self):
"""Test FalconH1Model.set_gguf_parameters writes vocab, attention head dims."""
from auto_round.export.export_to_gguf.conversion.falcon_h1 import FalconH1Model
obj = _make_mock_model(
FalconH1Model,
{
"vocab_size": 128256,
"max_position_embeddings": 131072,
"intermediate_size": 28672,
"num_attention_heads": 32,
"num_key_value_heads": 8,
"head_dim": 128,
"hidden_size": 5120,
"n_groups": 1,
"mamba_d_ssm": 8192,
"d_head": 64,
},
)
obj.rope_parameters = {"rope_theta": 10000.0}
# The __init__ method sets attributes like d_inner/d_head/n_group that
# are required for the post-assert in set_gguf_parameters.
obj.d_inner = 8192
obj.d_head = 64
obj.n_group = 1
with patch.object(FalconH1Model.__mro__[1], "set_gguf_parameters", lambda self: None):
obj.set_gguf_parameters()
w = obj.gguf_writer
w.add_vocab_size.assert_called_once_with(128256)
w.add_context_length.assert_called_once_with(131072)
w.add_feed_forward_length.assert_called_once_with(28672)
w.add_head_count.assert_called_once_with(32)
w.add_head_count_kv.assert_called_once_with(8)
w.add_key_length.assert_called_once_with(128)
w.add_value_length.assert_called_once_with(128)
w.add_rope_freq_base.assert_called_once_with(10000.0)
def test_modify_tensors_mlp_multipliers(self):
"""Test FalconH1Model.modify_tensors applies mlp_multipliers to down_proj/gate_proj."""
from auto_round.export.export_to_gguf.conversion.falcon_h1 import FalconH1Model
obj = _make_mock_model(FalconH1Model)
obj.mlp_multipliers = [0.5, 2.0]
obj.attention_in_multiplier = 1.0
obj.attention_out_multiplier = 1.0
obj.key_multiplier = 1.0
obj.ssm_in_multiplier = 1.0
obj.ssm_out_multiplier = 1.0
obj.n_group = 1
obj.d_inner = 64
# down_proj should be multiplied by mlp_multipliers[1] = 2.0
data = torch.ones(8, 8)
with patch.object(FalconH1Model.__mro__[1], "modify_tensors", lambda self, d, n, b: iter([(n, d)])):
results = list(obj.modify_tensors(data, "model.layers.0.mlp.down_proj.weight", bid=0))
assert torch.equal(results[0][1], torch.ones(8, 8) * 2.0)
# gate_proj should be multiplied by mlp_multipliers[0] = 0.5
with patch.object(FalconH1Model.__mro__[1], "modify_tensors", lambda self, d, n, b: iter([(n, d)])):
results = list(obj.modify_tensors(data, "model.layers.0.mlp.gate_proj.weight", bid=0))
assert torch.equal(results[0][1], torch.ones(8, 8) * 0.5)
def test_modify_tensors_attention_multipliers(self):
"""Test FalconH1Model.modify_tensors applies attention multipliers."""
from auto_round.export.export_to_gguf.conversion.falcon_h1 import FalconH1Model
obj = _make_mock_model(FalconH1Model)
obj.mlp_multipliers = [1.0, 1.0]
obj.attention_in_multiplier = 2.0
obj.attention_out_multiplier = 3.0
obj.key_multiplier = 4.0
obj.ssm_in_multiplier = 1.0
obj.ssm_out_multiplier = 1.0
obj.n_group = 1
obj.d_inner = 64
data = torch.ones(8, 8)
# q_proj gets attention_in_multiplier
with patch.object(FalconH1Model.__mro__[1], "modify_tensors", lambda self, d, n, b: iter([(n, d)])):
results = list(obj.modify_tensors(data, "model.layers.0.self_attn.q_proj.weight", bid=0))
assert torch.equal(results[0][1], torch.ones(8, 8) * 2.0)
# o_proj gets attention_out_multiplier
with patch.object(FalconH1Model.__mro__[1], "modify_tensors", lambda self, d, n, b: iter([(n, d)])):
results = list(obj.modify_tensors(data, "model.layers.0.self_attn.o_proj.weight", bid=0))
assert torch.equal(results[0][1], torch.ones(8, 8) * 3.0)
def test_modify_tensors_in_proj_ssm(self):
"""Test FalconH1Model.modify_tensors applies ssm_in + zxbcdt multipliers on in_proj."""
from auto_round.export.export_to_gguf.conversion.falcon_h1 import FalconH1Model
obj = _make_mock_model(
FalconH1Model,
{
"ssm_multipliers": [1.0, 1.0, 1.0, 1.0, 1.0],
"mamba_d_ssm": 8,
"mamba_n_groups": 1,
"mamba_d_state": 4,
},
)
obj.mlp_multipliers = [1.0, 1.0]
obj.attention_in_multiplier = 1.0
obj.attention_out_multiplier = 1.0
obj.key_multiplier = 1.0
obj.ssm_in_multiplier = 2.0
obj.ssm_out_multiplier = 1.0
obj.n_group = 1
obj.d_inner = 64
data = torch.ones(32, 4)
with patch.object(FalconH1Model.__mro__[1], "modify_tensors", lambda self, d, n, b: iter([(n, d)])):
results = list(obj.modify_tensors(data, "model.layers.0.mamba.in_proj.weight", bid=0))
# First all multiplied by ssm_in_multiplier = 2.0
assert torch.equal(results[0][1], torch.ones(32, 4) * 2.0)
def test_modify_tensors_lm_head_and_embed(self):
"""Test FalconH1Model.modify_tensors applies lm_head/embedding multipliers."""
from auto_round.export.export_to_gguf.conversion.falcon_h1 import FalconH1Model
obj = _make_mock_model(
FalconH1Model,
{
"lm_head_multiplier": 2.0,
"embedding_multiplier": 3.0,
},
)
obj.mlp_multipliers = [1.0, 1.0]
obj.attention_in_multiplier = 1.0
obj.attention_out_multiplier = 1.0
obj.key_multiplier = 1.0
obj.ssm_in_multiplier = 1.0
obj.ssm_out_multiplier = 1.0
obj.n_group = 1
obj.d_inner = 64
data = torch.ones(8, 8)
# lm_head
with patch.object(FalconH1Model.__mro__[1], "modify_tensors", lambda self, d, n, b: iter([(n, d)])):
results = list(obj.modify_tensors(data, "lm_head.weight", bid=None))
assert torch.equal(results[0][1], torch.ones(8, 8) * 2.0)
# embed_tokens
with patch.object(FalconH1Model.__mro__[1], "modify_tensors", lambda self, d, n, b: iter([(n, d)])):
results = list(obj.modify_tensors(data, "model.embed_tokens.weight", bid=None))
assert torch.equal(results[0][1], torch.ones(8, 8) * 3.0)
# ==============================================================================
# gpt_oss.py tests
# ==============================================================================
class TestGptOssConversion:
"""Tests for GPT-OSS conversion module."""
def test_set_vocab_uses_gpt2(self):
"""Test GptOssModel.set_vocab calls _set_vocab_gpt2."""
from auto_round.export.export_to_gguf.conversion.gpt_oss import GptOssModel
obj = _make_mock_model(GptOssModel)
with patch.object(obj, "_set_vocab_gpt2") as mock:
obj.set_vocab()
mock.assert_called_once_with()
def test_set_gguf_parameters(self):
"""Test GptOssModel.set_gguf_parameters writes sliding_window and expert FF length."""
from auto_round.export.export_to_gguf.conversion.gpt_oss import GptOssModel
obj = _make_mock_model(
GptOssModel,
{
"sliding_window": 128,
"intermediate_size": 2048,
"max_position_embeddings": 4096,
"hidden_size": 4096,
"num_attention_heads": 32,
},
)
with patch.object(GptOssModel.__mro__[1], "set_gguf_parameters", lambda self: None):
obj.set_gguf_parameters()
obj.gguf_writer.add_sliding_window.assert_called_once_with(128)
obj.gguf_writer.add_expert_feed_forward_length.assert_called_once_with(2048)
def test_filter_tensors_appends_weight_to_sinks(self):
"""Test GptOssModel.filter_tensors appends .weight to sinks tensors."""
from auto_round.export.export_to_gguf.conversion.gpt_oss import GptOssModel
def parent_filter(item):
return item
with patch.object(GptOssModel.__mro__[1], "filter_tensors", staticmethod(parent_filter)):
result = GptOssModel.filter_tensors(("model.layers.0.sinks", lambda: None))
assert result[0] == "model.layers.0.sinks.weight"
def test_transform_nibble_layout_runs(self):
"""Test transform_nibble_layout returns a uint8 tensor of the same shape."""
from auto_round.export.export_to_gguf.conversion.gpt_oss import GptOssModel
obj = _make_mock_model(GptOssModel)
# Single 16-element uint8 tensor with mixed nibbles
tensor = torch.tensor(
[[[[0x12, 0x34, 0x56, 0x78, 0x9A, 0xBC, 0xDE, 0xF0, 0x11, 0x22, 0x33, 0x44, 0x55, 0x66, 0x77, 0x88]]]],
dtype=torch.uint8,
)
out = obj.transform_nibble_layout(tensor)
assert out.shape == tensor.shape
assert out.dtype == torch.uint8
# ==============================================================================
# deci.py tests
# ==============================================================================
class TestDeciConversion:
"""Tests for DeciLM conversion module."""
def test_find_multiple(self):
"""Test DeciModel._find_multiple rounds up to nearest multiple of k."""
from auto_round.export.export_to_gguf.conversion.deci import DeciModel
# 128 % 256 != 0 -> round up to 256 (uses k - n%k = 256 - 128)
assert DeciModel._find_multiple(128, 256) == 256
# 100 % 256 != 0 -> round up to 256
assert DeciModel._find_multiple(100, 256) == 256
# 257 % 256 != 0 -> round up to 512
assert DeciModel._find_multiple(257, 256) == 512
def test_permute_preserves_shape(self):
"""Test DeciModel.permute preserves tensor shape."""
from auto_round.export.export_to_gguf.conversion.deci import DeciModel
weights = torch.randn(4096, 1024)
out = DeciModel.permute(weights, n_head=32, n_head_kv=32)
assert out.shape == weights.shape
# ==============================================================================
# llada.py tests
# ==============================================================================
class TestLladaConversion:
"""Tests for LLaDA conversion module."""
def test_llada_set_gguf_parameters(self):
"""Test LLaDAModel.set_gguf_parameters writes non-causal + diffusion shift flags."""
from auto_round.export.export_to_gguf.conversion.llada import LLaDAModel
obj = _make_mock_model(
LLaDAModel,
{
"vocab_size": 126336,
"head_dim": 128,
"num_attention_heads": 32,
"max_sequence_length": 4096,
"d_model": 4096,
"mlp_hidden_size": 12288,
"max_position_embeddings": 4096,
"hidden_size": 4096,
"intermediate_size": 12288,
},
)
with patch.object(LLaDAModel.__mro__[1], "set_gguf_parameters", lambda self: None):
obj.set_gguf_parameters()
w = obj.gguf_writer
w.add_vocab_size.assert_called_once_with(126336)
w.add_rope_dimension_count.assert_called_once_with(128)
w.add_causal_attention.assert_called_once_with(False)
w.add_diffusion_shift_logits.assert_called_once_with(False)
def test_llada_moe_set_gguf_parameters(self):
"""Test LLaDAMoEModel.set_gguf_parameters writes expert FF + mask token."""
from auto_round.export.export_to_gguf.conversion.llada import LLaDAMoEModel
obj = _make_mock_model(
LLaDAMoEModel,
{
"expert_intermediate_size": 2048,
"max_position_embeddings": 4096,
"hidden_size": 4096,
"intermediate_size": 12288,
"num_attention_heads": 32,
},
)
with patch.object(LLaDAMoEModel.__mro__[1], "set_gguf_parameters", lambda self: None):
obj.set_gguf_parameters()
obj.gguf_writer.add_expert_feed_forward_length.assert_called_once_with(2048)
obj.gguf_writer.add_mask_token_id.assert_called_once_with(156895)
obj.gguf_writer.add_causal_attention.assert_called_once_with(False)
# ==============================================================================
# kimi_linear.py tests
# ==============================================================================
class TestKimiLinearConversion:
"""Tests for KimiLinear conversion module."""
def test_set_gguf_parameters_with_mla(self):
"""Test KimiLinearModel.set_gguf_parameters writes MLA + KDA parameters."""
from auto_round.export.export_to_gguf.conversion.kimi_linear import KimiLinearModel
obj = _make_mock_model(
KimiLinearModel,
{
"vocab_size": 102400,
"num_hidden_layers": 24,
"linear_attn_config": {
"full_attn_layers": [12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23],
"short_conv_kernel_size": 4,
"head_dim": 128,
},
"q_lora_rank": 1536,
"kv_lora_rank": 512,
"qk_nope_head_dim": 128,
"qk_rope_head_dim": 64,
"v_head_dim": 128,
"moe_intermediate_size": 1024,
"num_shared_experts": 1,
"first_k_dense_replace": 1,
"routed_scaling_factor": 2.446,
"n_embd_head_k_mla": 192,
"n_embd_head_v_mla": 128,
"max_position_embeddings": 32768,
"hidden_size": 4096,
"intermediate_size": 12288,
"num_attention_heads": 32,
},
)
with patch.object(KimiLinearModel.__mro__[1], "set_gguf_parameters", lambda self: None):
obj.set_gguf_parameters()
w = obj.gguf_writer
w.add_vocab_size.assert_called_once_with(102400)
w.add_q_lora_rank.assert_called_once_with(1536)
w.add_kv_lora_rank.assert_called_once_with(512)
# key_length = kv_lora_rank + qk_rope_head_dim = 512 + 64 = 576
w.add_key_length.assert_called_once_with(576)
w.add_key_length_mla.assert_called_once_with(192)
w.add_value_length_mla.assert_called_once_with(128)
w.add_rope_dimension_count.assert_called_once_with(64)
w.add_expert_feed_forward_length.assert_called_once_with(1024)
w.add_expert_shared_count.assert_called_once_with(1)
w.add_leading_dense_block_count.assert_called_once_with(1)
w.add_expert_weights_scale.assert_called_once_with(2.446)
w.add_ssm_conv_kernel.assert_called_once_with(4)
w.add_kda_head_dim.assert_called_once_with(128)
def test_modify_tensors_conv1d_reshape(self):
"""Test KimiLinearModel.modify_tensors reshapes 2D conv1d weights to (1, d_inner, 1, d_conv)."""
from auto_round.export.export_to_gguf.conversion.kimi_linear import KimiLinearModel
obj = _make_mock_model(
KimiLinearModel,
{
"num_local_experts": 2,
},
)
# 2D weight [d_inner, d_conv]
data = torch.randn(8, 4)
with patch.object(KimiLinearModel.__mro__[1], "modify_tensors", lambda self, d, n, b: iter([(n, d)])):
result = list(obj.modify_tensors(data, "model.layers.0.linear_attn.q_conv1d.weight", bid=0))
# Reshape to (1, d_inner, 1, d_conv) = (1, 8, 1, 4)
assert result[0][1].shape == (1, 8, 1, 4)
def test_modify_tensors_conv1d_3d_reshape(self):
"""Test KimiLinearModel.modify_tensors reshapes 3D conv1d weights."""
from auto_round.export.export_to_gguf.conversion.kimi_linear import KimiLinearModel
obj = _make_mock_model(
KimiLinearModel,
{
"num_local_experts": 2,
},
)
# 3D weight [d_inner, 1, d_conv]
data = torch.randn(8, 1, 4)
with patch.object(KimiLinearModel.__mro__[1], "modify_tensors", lambda self, d, n, b: iter([(n, d)])):
result = list(obj.modify_tensors(data, "model.layers.0.linear_attn.k_conv1d.weight", bid=0))
assert result[0][1].shape == (1, 8, 1, 4)
def test_modify_tensors_a_log_negation(self):
"""Test KimiLinearModel.modify_tensors negates exp of A_log."""
from auto_round.export.export_to_gguf.conversion.kimi_linear import KimiLinearModel
obj = _make_mock_model(
KimiLinearModel,
{
"num_local_experts": 2,
},
)
data = torch.tensor([0.0, 1.0])
with patch.object(KimiLinearModel.__mro__[1], "modify_tensors", lambda self, d, n, b: iter([(n, d)])):
result = list(obj.modify_tensors(data, "model.layers.0.linear_attn.A_log", bid=0))
assert torch.allclose(result[0][1], torch.tensor([-1.0, -2.7183]), atol=1e-3)
def test_modify_tensors_dt_bias_rename(self):
"""Test KimiLinearModel.modify_tensors renames dt_bias to dt_proj.bias."""
from auto_round.export.export_to_gguf.conversion.kimi_linear import KimiLinearModel
obj = _make_mock_model(
KimiLinearModel,
{
"num_local_experts": 2,
},
)
captured = []
with patch.object(
KimiLinearModel.__mro__[1], "modify_tensors", lambda self, d, n, b: captured.append(n) or iter([(n, d)])
):
data = torch.randn(4)
list(obj.modify_tensors(data, "model.layers.0.linear_attn.dt_bias", bid=0))
# The name should be renamed from dt_bias to dt_proj.bias
assert any("dt_proj.bias" in n for n in captured)
def test_modify_tensors_experts_merge(self):
"""Test KimiLinearModel.modify_tensors merges block_sparse_moe.experts tensors."""
from auto_round.export.export_to_gguf.conversion.kimi_linear import KimiLinearModel
obj = _make_mock_model(
KimiLinearModel,
{
"num_local_experts": 2,
"num_key_value_heads": 1,
},
)
captured = []
with patch.object(
KimiLinearModel.__mro__[1], "modify_tensors", lambda self, d, n, b: captured.append(n) or iter([(n, d)])
):
# 2 experts * 3 weights = 6 tensors trigger merge
for xid in range(2):
for wid in ["w1", "w2", "w3"]:
ename = f"model.layers.0.block_sparse_moe.experts.{xid}.{wid}.weight"
list(obj.modify_tensors(torch.randn(8, 8), ename, bid=0))
# 3 merged names expected (FFN_GATE_EXP / FFN_DOWN_EXP / FFN_UP_EXP)
assert len(captured) == 3
def test_modify_tensors_kv_b_split(self):
"""Test KimiLinearModel.modify_tensors splits kv_b_proj.weight into k_b/v_b."""
from auto_round.export.export_to_gguf.conversion.kimi_linear import KimiLinearModel
obj = _make_mock_model(
KimiLinearModel,
{
"num_local_experts": 2,
"num_key_value_heads": 4,
"v_head_dim": 64,
"qk_nope_head_dim": 32,
"q_lora_rank": 0,
},
)
captured = []
# kv_b_proj shape: [n_head_kv * (v_head_dim + qk_nope_head_dim), hidden_in]
# = 4 * (64 + 32) = 384
data = torch.randn(384, 8)
with patch.object(
KimiLinearModel.__mro__[1], "modify_tensors", lambda self, d, n, b: captured.append(n) or iter([(n, d)])
):
list(obj.modify_tensors(data, "model.layers.0.self_attn.kv_b_proj.weight", bid=0))
# Should yield 2 outputs: k_b_proj and v_b_proj
assert len(captured) == 2
assert any("k_b_proj" in n for n in captured)
assert any("v_b_proj" in n for n in captured)
# ==============================================================================
# arctic.py tests
# ==============================================================================
class TestArcticConversion:
"""Tests for Arctic conversion module."""
def test_set_gguf_parameters(self):
"""Test ArcticModel.set_gguf_parameters writes vocab_size and rope_dimension_count."""
from auto_round.export.export_to_gguf.conversion.arctic import ArcticModel
obj = _make_mock_model(
ArcticModel,
{
"vocab_size": 100352,
"hidden_size": 7168,
"num_attention_heads": 56,
"max_position_embeddings": 4096,
"intermediate_size": 18432,
},
)
with patch.object(ArcticModel.__mro__[1], "set_gguf_parameters", lambda self: None):
obj.set_gguf_parameters()
w = obj.gguf_writer
w.add_vocab_size.assert_called_once_with(100352)
# 7168 / 56 = 128
w.add_rope_dimension_count.assert_called_once_with(128)
def test_modify_tensors_qk_permute(self):
"""Test ArcticModel.modify_tensors permutes q_proj/k_proj via LlamaModel.permute."""
from auto_round.export.export_to_gguf.conversion.arctic import ArcticModel
obj = _make_mock_model(
ArcticModel,
{
"num_attention_heads": 32,
"num_key_value_heads": 8,
},
)
# q_proj gets permuted by LlamaModel.permute
q_data = torch.randn(4096, 1024)
q_result = list(obj.modify_tensors(q_data, "model.layers.0.self_attn.q_proj.weight", bid=0))
# After permute the shape is preserved
assert q_result[0][1].shape == q_data.shape
# Non-q/k tensors just pass through
emb = torch.randn(100352, 7168)
emb_result = list(obj.modify_tensors(emb, "model.embed_tokens.weight", bid=None))
assert torch.equal(emb_result[0][1], emb)
def test_modify_tensors_expert_merging(self):
"""Test ArcticModel.modify_tensors merges block_sparse_moe.experts tensors."""
from auto_round.export.export_to_gguf.conversion.arctic import ArcticModel
obj = _make_mock_model(
ArcticModel,
{
"num_attention_heads": 32,
"num_key_value_heads": 8,
"num_local_experts": 2,
},
)
# Feed all expert tensors (3 per expert = 6 total) for bid=0
n_experts = 2
captured = []
with patch.object(
ArcticModel.__mro__[1], "modify_tensors", lambda self, d, n, b: captured.append((n, d)) or iter([(n, d)])
):
for xid in range(n_experts):
for wid in ["w1", "w2", "w3"]:
ename = f"model.layers.0.block_sparse_moe.experts.{xid}.{wid}.weight"
list(obj.modify_tensors(torch.randn(8, 8), ename, bid=0))
# After all 6 tensors, we should get 3 merged tensors (w1/w2/w3 each stacked)
merged_names = [n for n, _ in captured]
assert "layers.0.feed_forward.experts.w1.weight" in merged_names
assert "layers.0.feed_forward.experts.w2.weight" in merged_names
assert "layers.0.feed_forward.experts.w3.weight" in merged_names
# ==============================================================================
# bailingmoe.py tests
# ==============================================================================
class TestBailingMoeConversion:
"""Tests for BailingMoe / BailingMoeV2 conversion module."""
def test_bailingmoe_set_gguf_parameters(self):
"""Test BailingMoeModel.set_gguf_parameters writes MoE + first_k_dense_replace."""
from auto_round.export.export_to_gguf.conversion.bailingmoe import BailingMoeModel
obj = _make_mock_model(
BailingMoeModel,
{
"vocab_size": 107136,
"head_dim": 128,
"hidden_size": 4096,
"num_attention_heads": 32,
"first_k_dense_replace": 1,
"moe_intermediate_size": 1536,
"num_shared_experts": 2,
"norm_topk_prob": True,
"max_position_embeddings": 131072,
"intermediate_size": 12288,
},
)
with patch.object(BailingMoeModel.__mro__[1], "set_gguf_parameters", lambda self: None):
obj.set_gguf_parameters()
w = obj.gguf_writer
w.add_rope_dimension_count.assert_called_once_with(128)
w.add_leading_dense_block_count.assert_called_once_with(1)
w.add_vocab_size.assert_called_once_with(107136)
w.add_expert_feed_forward_length.assert_called_once_with(1536)
w.add_expert_weights_scale.assert_called_once_with(1.0)
w.add_expert_shared_count.assert_called_once_with(2)
w.add_expert_weights_norm.assert_called_once_with(True)
def test_bailingmoe_v2_set_gguf_parameters(self):
"""Test BailingMoeV2Model.set_gguf_parameters handles partial_rotary_factor + nextn."""
from auto_round.export.export_to_gguf.conversion.bailingmoe import BailingMoeV2Model
obj = _make_mock_model(
BailingMoeV2Model,
{
"vocab_size": 107136,
"hidden_size": 4096,
"num_attention_heads": 32,
"head_dim": 128,
"first_k_dense_replace": 1,
"moe_intermediate_size": 1536,
"moe_shared_expert_intermediate_size": 256,
"routed_scaling_factor": 2.5,
"num_shared_experts": 2,
"norm_topk_prob": True,
"num_nextn_predict_layers": 1,
"partial_rotary_factor": 0.5,
"max_position_embeddings": 131072,
"intermediate_size": 12288,
},
)
with patch.object(BailingMoeV2Model.__mro__[1], "set_gguf_parameters", lambda self: None):
obj.set_gguf_parameters()
w = obj.gguf_writer
# partial_rotary_factor * head_dim = 0.5 * 128 = 64
w.add_rope_dimension_count.assert_called_once_with(64)
w.add_leading_dense_block_count.assert_called_once_with(1)
w.add_expert_shared_feed_forward_length.assert_called_once_with(256)
w.add_expert_weights_scale.assert_called_once_with(2.5)
w.add_nextn_predict_layers.assert_called_once_with(1)
def test_bailingmoe_permute_static(self):
"""Test BailingMoeModel.permute preserves tensor shape."""
from auto_round.export.export_to_gguf.conversion.bailingmoe import BailingMoeModel
weights = torch.randn(4096, 1024)
out = BailingMoeModel.permute(weights, n_head=32, n_head_kv=32)
assert out.shape == weights.shape
def test_bailingmoe_modify_tensors_dense(self):
"""Test BailingMoeModel.modify_tensors splits qkv + dense rename."""
from auto_round.export.export_to_gguf.conversion.bailingmoe import BailingMoeModel
obj = _make_mock_model(
BailingMoeModel,
{
"num_attention_heads": 32,
"num_key_value_heads": 8,
"head_dim": 128,
"hidden_size": 4096,
},
)
captured = []
with patch.object(
BailingMoeModel.__mro__[1],
"modify_tensors",
lambda self, d, n, b: captured.append((n, d)) or iter([(n, d)]),
):
# attention.dense.weight -> ATTN_OUT
data = torch.randn(4096, 4096)
list(obj.modify_tensors(data, "model.layers.0.attention.dense.weight", bid=0))
# Should be renamed via format_tensor_name(gguf.MODEL_TENSOR.ATTN_OUT, bid)
assert len(captured) == 1
def test_bailingmoe_modify_tensors_qkv_split(self):
"""Test BailingMoeModel.modify_tensors splits query_key_value.weight into q/k/v."""
from auto_round.export.export_to_gguf.conversion.bailingmoe import BailingMoeModel
obj = _make_mock_model(
BailingMoeModel,
{
"num_attention_heads": 32,
"num_key_value_heads": 8,
"head_dim": 128,
"hidden_size": 4096,
},
)
captured = []
with patch.object(
BailingMoeModel.__mro__[1],
"modify_tensors",
lambda self, d, n, b: captured.append((n, d)) or iter([(n, d)]),
):
# total dim = (32 + 2*8) * 128 = 6144
data = torch.randn(6144, 4096)
list(obj.modify_tensors(data, "model.layers.0.attention.query_key_value.weight", bid=0))
# Should split into 3 outputs
assert len(captured) == 3
# ==============================================================================
# exaone.py tests
# ==============================================================================
class TestExaoneConversion:
"""Tests for Exaone conversion module."""
def test_set_gguf_parameters(self):
"""Test ExaoneModel.set_gguf_parameters writes rope dim from partial_rotary_factor."""
from auto_round.export.export_to_gguf.conversion.exaone import ExaoneModel
obj = _make_mock_model(
ExaoneModel,
{
"activation_function": "silu",
"partial_rotary_factor": 0.5,
"hidden_size": 4096,
"num_attention_heads": 32,
"max_position_embeddings": 32768,
"intermediate_size": 16384,
},
)
with patch.object(ExaoneModel.__mro__[1], "set_gguf_parameters", lambda self: None):
obj.set_gguf_parameters()
# 0.5 * (4096 / 32) = 0.5 * 128 = 64
obj.gguf_writer.add_rope_dimension_count.assert_called_once_with(64)
# ==============================================================================
# internlm.py tests
# ==============================================================================
class TestInternlmConversion:
"""Tests for InternLM2 conversion module."""
def test_filter_tensors_drops_mlp_and_vision(self):
"""Test InternLM2Model.filter_tensors drops tensors whose name starts with 'mlp' or 'vision_model'."""
from auto_round.export.export_to_gguf.conversion.internlm import InternLM2Model
# Names starting with "mlp" or "vision_model" should be dropped
assert InternLM2Model.filter_tensors(("mlp.down_proj.weight", lambda: None)) is None
assert InternLM2Model.filter_tensors(("vision_model.encoder.weight", lambda: None)) is None
def test_modify_tensors_q_permute(self):
"""Test InternLM2Model.modify_tensors permutes q_proj via LlamaModel.permute."""
from auto_round.export.export_to_gguf.conversion.internlm import InternLM2Model
obj = _make_mock_model(
InternLM2Model,
{
"num_attention_heads": 32,
"num_key_value_heads": 8,
"hidden_size": 4096,
},
)
data = torch.randn(4096, 1024)
with patch.object(InternLM2Model.__mro__[1], "modify_tensors", lambda self, d, n, b: iter([(n, d)])):
result = list(obj.modify_tensors(data, "model.layers.0.attention.wq.weight", bid=0))
# Permute preserves shape
assert result[0][1].shape == data.shape
# ==============================================================================
# glm.py tests
# ==============================================================================
class TestGlm4Conversion:
"""Tests for GLM4 conversion module."""
def test_set_gguf_parameters_with_head_dim(self):
"""Test Glm4Model.set_gguf_parameters writes rope dim using head_dim * partial_rotary_factor."""
from auto_round.export.export_to_gguf.conversion.glm import Glm4Model
obj = _make_mock_model(
Glm4Model,
{
"head_dim": 128,
"partial_rotary_factor": 0.5,
"hidden_size": 4096,
"num_attention_heads": 32,
"max_position_embeddings": 131072,
"intermediate_size": 13696,
},
)
# rope_parameters dict (used in __init__) needs to be present on the object
obj.rope_parameters = {"partial_rotary_factor": 0.5}
# Use the already-instantiated obj.partial_rotary_factor (set in __init__)
obj.partial_rotary_factor = 0.5
with patch.object(Glm4Model.__mro__[1], "set_gguf_parameters", lambda self: None):
obj.set_gguf_parameters()
# int(128 * 0.5) = 64
obj.gguf_writer.add_rope_dimension_count.assert_called_once_with(64)
# ==============================================================================
# lfm2.py tests
# ==============================================================================
class TestLfm2Conversion:
"""Tests for LFM2 conversion module."""
def test_lfm2_set_gguf_parameters(self):
"""Test LFM2Model.set_gguf_parameters writes vocab + lfm2-specific fields."""
from auto_round.export.export_to_gguf.conversion.lfm2 import LFM2Model
obj = _make_mock_model(
LFM2Model,
{
"vocab_size": 65536,
"layer_types": ["conv", "full_attention"],
"conv_L_cache": 4,
"norm_eps": 1e-5,
"block_ff_dim": 8192,
"block_auto_adjust_ff_dim": False,
"block_ffn_dim_multiplier": None,
"block_multiple_of": 256,
"intermediate_size": 8192,
"num_key_value_heads": 8,
"max_position_embeddings": 32768,
"hidden_size": 2048,
"num_attention_heads": 32,
},
)
with patch.object(LFM2Model.__mro__[1], "set_gguf_parameters", lambda self: None):
obj.set_gguf_parameters()
w = obj.gguf_writer
w.add_vocab_size.assert_called_once_with(65536)
w.add_shortconv_l_cache.assert_called_once_with(4)
w.add_layer_norm_rms_eps.assert_called_once_with(1e-5)
# num_kv_heads should be set: [0, 8] (0 for conv, 8 for full_attention)
assert obj.hparams["num_key_value_heads"] == [0, 8]
def test_lfm2moe_set_gguf_parameters(self):
"""Test LFM2MoeModel.set_gguf_parameters writes MoE fields and gating func."""
from auto_round.export.export_to_gguf.conversion.base import gguf
from auto_round.export.export_to_gguf.conversion.lfm2 import LFM2MoeModel
obj = _make_mock_model(
LFM2MoeModel,
{
"vocab_size": 65536,
"layer_types": ["conv", "full_attention"],
"conv_L_cache": 4,
"moe_intermediate_size": 2048,
"num_dense_layers": 1,
"num_local_experts": 32,
"max_position_embeddings": 32768,
"hidden_size": 2048,
"intermediate_size": 8192,
"num_attention_heads": 32,
"num_key_value_heads": 8,
},
)
with patch.object(LFM2MoeModel.__mro__[1], "set_gguf_parameters", lambda self: None):
obj.set_gguf_parameters()
w = obj.gguf_writer
w.add_expert_feed_forward_length.assert_called_once_with(2048)
w.add_leading_dense_block_count.assert_called_once_with(1)
w.add_expert_gating_func.assert_called_once_with(gguf.ExpertGatingFuncType.SIGMOID)
w.add_shortconv_l_cache.assert_called_once_with(4)
def test_lfm2_modify_tensors_conv_squeeze(self):
"""Test LFM2Model.modify_tensors squeezes dim 1 of conv.conv weights."""
from auto_round.export.export_to_gguf.conversion.lfm2 import LFM2Model
obj = _make_mock_model(LFM2Model)
data = torch.randn(8, 1, 4) # has a length-1 dim at position 1
with patch.object(LFM2Model.__mro__[1], "modify_tensors", lambda self, d, n, b: iter([(n, d)])):
result = list(obj.modify_tensors(data, "model.layers.0.conv.conv.weight", bid=0))
# After squeeze(1), shape becomes (8, 4)
assert result[0][1].shape == (8, 4)
def test_lfm2moe_modify_tensors_experts_merge(self):
"""Test LFM2MoeModel.modify_tensors merges feed_forward.experts tensors."""
from auto_round.export.export_to_gguf.conversion.lfm2 import LFM2MoeModel
obj = _make_mock_model(
LFM2MoeModel,
{
"num_local_experts": 2,
},
)
captured = []
with patch.object(
LFM2MoeModel.__mro__[1], "modify_tensors", lambda self, d, n, b: captured.append(n) or iter([(n, d)])
):
# 2 experts * 3 weights = 6 tensors trigger merge
for xid in range(2):
for wid in ["w1", "w2", "w3"]:
ename = f"model.layers.0.feed_forward.experts.{xid}.{wid}.weight"
list(obj.modify_tensors(torch.randn(8, 8), ename, bid=0))
assert any("experts.w1.weight" in n for n in captured)
assert any("experts.w2.weight" in n for n in captured)
assert any("experts.w3.weight" in n for n in captured)
# ==============================================================================
# gemma.py tests
# ==============================================================================
class TestGemmaConversion:
"""Tests for Gemma / Gemma2 / Gemma3 conversion module."""
def test_gemma_set_gguf_parameters(self):
"""Test GemmaModel.set_gguf_parameters writes gemma fields."""
from auto_round.export.export_to_gguf.conversion.gemma import GemmaModel
obj = _make_mock_model(
GemmaModel,
{
"max_position_embeddings": 8192,
"hidden_size": 2048,
"intermediate_size": 16384,
"num_attention_heads": 8,
"num_key_value_heads": 1,
"head_dim": 256,
"rms_norm_eps": 1e-6,
},
)
obj.set_gguf_parameters()
w = obj.gguf_writer
w.add_context_length.assert_called_once_with(8192)
w.add_embedding_length.assert_called_once_with(2048)
w.add_feed_forward_length.assert_called_once_with(16384)
w.add_head_count.assert_called_once_with(8)
w.add_head_count_kv.assert_called_once_with(1)
w.add_layer_norm_rms_eps.assert_called_once_with(1e-6)
w.add_key_length.assert_called_once_with(256)
w.add_value_length.assert_called_once_with(256)
def test_gemma_set_gguf_parameters_defaults_kv_to_heads(self):
"""Test GemmaModel.set_gguf_parameters defaults num_key_value_heads to num_attention_heads."""
from auto_round.export.export_to_gguf.conversion.gemma import GemmaModel
obj = _make_mock_model(
GemmaModel,
{
"max_position_embeddings": 8192,
"hidden_size": 2048,
"intermediate_size": 16384,
"num_attention_heads": 8,
# No num_key_value_heads provided
"head_dim": 256,
"rms_norm_eps": 1e-6,
},
)
obj.set_gguf_parameters()
obj.gguf_writer.add_head_count_kv.assert_called_once_with(8)
def test_gemma_modify_tensors_norm_plus_one(self):
"""Test GemmaModel.modify_tensors adds 1.0 to .norm.weight."""
from auto_round.export.export_to_gguf.conversion.gemma import GemmaModel
obj = _make_mock_model(GemmaModel, {})
data = torch.zeros(8)
with patch.object(GemmaModel.__mro__[1], "modify_tensors", lambda self, d, n, b: iter([(n, d)])):
result = list(obj.modify_tensors(data, "model.layers.0.input_layernorm.weight", bid=0))
# norm.weight should be incremented by 1.0
assert torch.equal(result[0][1], torch.ones(8))
# Non-norm tensor is passed through unchanged
result2 = list(obj.modify_tensors(data, "model.layers.0.self_attn.q_proj.weight", bid=0))
assert torch.equal(result2[0][1], torch.zeros(8))
def test_gemma2_set_gguf_parameters_softcap(self):
"""Test Gemma2Model.set_gguf_parameters writes softcap and sliding_window."""
from auto_round.export.export_to_gguf.conversion.gemma import Gemma2Model
obj = _make_mock_model(
Gemma2Model,
{
"max_position_embeddings": 8192,
"hidden_size": 3072,
"intermediate_size": 24576,
"num_attention_heads": 16,
"num_key_value_heads": 8,
"head_dim": 256,
"rms_norm_eps": 1e-6,
"attn_logit_softcapping": 50.0,
"final_logit_softcapping": 30.0,
"sliding_window": 4096,
},
)
obj.set_gguf_parameters()
w = obj.gguf_writer
w.add_attn_logit_softcapping.assert_called_once_with(50.0)
w.add_final_logit_softcapping.assert_called_once_with(30.0)
w.add_sliding_window.assert_called_once_with(4096)
def test_gemma3_set_gguf_parameters(self):
"""Test Gemma3Model.set_gguf_parameters writes gemma3 fields + asserts attn_softcapping is None."""
from auto_round.export.export_to_gguf.conversion.gemma import Gemma3Model
obj = _make_mock_model(
Gemma3Model,
{
"max_position_embeddings": 131072,
"hidden_size": 2560,
"intermediate_size": 10240,
"num_attention_heads": 10,
"num_key_value_heads": 4,
"head_dim": 256,
"rms_norm_eps": 1e-6,
"attn_logit_softcapping": None,
"final_logit_softcapping": 0.0,
"sliding_window": 512,
"sliding_window_pattern": 6, # != 1, so add_sliding_window is called
},
)
obj.rope_parameters = {"rope_theta": 1_000_000.0}
with patch.object(Gemma3Model.__mro__[1], "set_gguf_parameters", lambda self: None):
obj.set_gguf_parameters()
w = obj.gguf_writer
w.add_context_length.assert_called_once_with(131072)
w.add_head_count.assert_called_once_with(10)
w.add_layer_norm_rms_eps.assert_called_once_with(1e-6)
w.add_key_length.assert_called_once_with(256)
w.add_value_length.assert_called_once_with(256)
w.add_rope_freq_base.assert_called_once_with(1_000_000.0)
w.add_head_count_kv.assert_called_once_with(4)
w.add_sliding_window.assert_called_once_with(512)
# final_logit_softcapping=0.0 is falsy, so add_final_logit_softcapping not called
w.add_final_logit_softcapping.assert_not_called()
def test_gemma3_set_gguf_parameters_no_sliding_window(self):
"""Test Gemma3Model.set_gguf_parameters skips sliding_window when pattern == 1."""
from auto_round.export.export_to_gguf.conversion.gemma import Gemma3Model
obj = _make_mock_model(
Gemma3Model,
{
"max_position_embeddings": 131072,
"hidden_size": 2560,
"intermediate_size": 10240,
"num_attention_heads": 10,
"num_key_value_heads": 4,
"head_dim": 256,
"rms_norm_eps": 1e-6,
"attn_logit_softcapping": None,
"sliding_window": 512,
"sliding_window_pattern": 1,
},
)
obj.rope_parameters = {"rope_theta": 1_000_000.0}
obj.set_gguf_parameters()
obj.gguf_writer.add_sliding_window.assert_not_called()
def test_gemma3_norm_shift(self):
"""Test Gemma3Model.norm_shift returns 1.0 for norm.weight, else 0.0."""
from auto_round.export.export_to_gguf.conversion.gemma import Gemma3Model
# norm_shift is a plain instance method; call it on a bare object
obj = Gemma3Model.__new__(Gemma3Model)
assert obj.norm_shift("model.layers.0.input_layernorm.weight") == 1.0
assert obj.norm_shift("model.embed_tokens.weight") == 0.0
def test_gemma3_modify_tensors_norm_shift(self):
"""Test Gemma3Model.modify_tensors adds norm_shift to .norm.weight tensors."""
from auto_round.export.export_to_gguf.conversion.gemma import Gemma3Model
obj = _make_mock_model(Gemma3Model, {})
data = torch.zeros(8)
with patch.object(Gemma3Model.__mro__[1], "modify_tensors", lambda self, d, n, b: iter([(n, d)])):
result = list(obj.modify_tensors(data, "model.layers.0.input_layernorm.weight", bid=0))
# norm_shift = 1.0 for .norm.weight
assert torch.equal(result[0][1], torch.ones(8))
def test_gemma3_vision_set_gguf_parameters(self):
"""Test Gemma3VisionModel.set_gguf_parameters writes GEMMA3 projector + use_gelu."""
from auto_round.export.export_to_gguf.conversion.base import gguf
from auto_round.export.export_to_gguf.conversion.gemma import Gemma3VisionModel
obj = _make_mock_model(
Gemma3VisionModel,
{
"layer_norm_eps": 1e-6,
"image_size": 896,
"patch_size": 14,
},
)
obj.preprocessor_config = {"image_seq_length": 256}
with patch.object(Gemma3VisionModel.__mro__[1], "set_gguf_parameters", lambda self: None):
obj.set_gguf_parameters()
w = obj.gguf_writer
w.add_clip_projector_type.assert_called_once_with(gguf.VisionProjectorType.GEMMA3)
w.add_vision_attention_layernorm_eps.assert_called_once_with(1e-6)
w.add_vision_use_gelu.assert_called_once_with(True)
# proj_scale_factor = (896 // 14) // 16 = 64 // 16 = 4 -> default, not written
def test_gemma3_vision_tensor_force_quant(self):
"""Test Gemma3VisionModel.tensor_force_quant forces F16/F32 for input_projection/embeddings."""
from auto_round.export.export_to_gguf.conversion.base import gguf
from auto_round.export.export_to_gguf.conversion.gemma import Gemma3VisionModel
obj = Gemma3VisionModel.__new__(Gemma3VisionModel)
# input_projection forces F16
quant = obj.tensor_force_quant("model.vision_tower.input_projection.weight", "v.input.weight", bid=0, n_dims=2)
assert quant == gguf.GGMLQuantizationType.F16
# embeddings forces F32
quant = obj.tensor_force_quant("model.vision_tower.embeddings.weight", "v.embed.weight", bid=0, n_dims=2)
assert quant == gguf.GGMLQuantizationType.F32
def test_gemma3_vision_filter_tensors(self):
"""Test Gemma3VisionModel.filter_tensors drops non-vision tensors."""
from auto_round.export.export_to_gguf.conversion.gemma import Gemma3VisionModel
# Non-vision tensors are dropped
assert Gemma3VisionModel.filter_tensors(("model.embed_tokens.weight", lambda: None)) is None
assert Gemma3VisionModel.filter_tensors(("model.layers.0.self_attn.q_proj.weight", lambda: None)) is None
# ==============================================================================
# kimivl.py tests
# ==============================================================================
class TestKimiVlConversion:
"""Tests for KimiVL conversion module."""
def test_set_gguf_parameters(self):
"""Test KimiVLModel.set_gguf_parameters writes vision projector type and layernorm eps."""
from auto_round.export.export_to_gguf.conversion.base import gguf
from auto_round.export.export_to_gguf.conversion.kimivl import KimiVLModel
obj = _make_mock_model(KimiVLModel)
obj.hparams_vision = {"layer_norm_eps": 1e-5}
# image_size is set in __init__ from hparams_vision
with patch.object(KimiVLModel.__mro__[1], "set_gguf_parameters", lambda self: None):
obj.set_gguf_parameters()
w = obj.gguf_writer
w.add_clip_projector_type.assert_called_once_with(gguf.VisionProjectorType.KIMIVL)
w.add_vision_use_gelu.assert_called_once_with(True)
w.add_vision_projector_scale_factor.assert_called_once_with(2)
w.add_vision_attention_layernorm_eps.assert_called_once_with(1e-5)
def test_filter_tensors_drops_non_vision(self):
"""Test KimiVLModel.filter_tensors drops non-vision tensors."""
from auto_round.export.export_to_gguf.conversion.kimivl import KimiVLModel
# Non-vision tensors should be dropped
assert KimiVLModel.filter_tensors(("model.embed_tokens.weight", lambda: None)) is None
assert KimiVLModel.filter_tensors(("model.layers.0.input_layernorm.weight", lambda: None)) is None
# ==============================================================================
# phi.py tests
# ==============================================================================
class TestPhiConversion:
"""Tests for Phi conversion module."""
def test_phi2_set_gguf_parameters(self):
"""Test Phi2Model.set_gguf_parameters writes phi2 fields."""
from auto_round.export.export_to_gguf.conversion.phi import Phi2Model
obj = _make_mock_model(
Phi2Model,
{
"partial_rotary_factor": 0.5,
"hidden_size": 2560,
"num_attention_heads": 32,
"n_positions": 2048,
"layer_norm_epsilon": 1e-5,
"max_position_embeddings": 2048,
"intermediate_size": 10240,
},
)
obj.set_gguf_parameters()
w = obj.gguf_writer
w.add_context_length.assert_called_once_with(2048)
w.add_embedding_length.assert_called_once_with(2560)
w.add_feed_forward_length.assert_called_once_with(10240)
w.add_head_count.assert_called_once_with(32)
w.add_head_count_kv.assert_called_once_with(32)
w.add_layer_norm_eps.assert_called_once_with(1e-5)
w.add_rope_dimension_count.assert_called_once_with(40) # int(0.5 * 2560) // 32 = 1280 // 32 = 40
w.add_add_bos_token.assert_called_once_with(False)
def test_phi3_set_gguf_parameters(self):
"""Test Phi3MiniModel.set_gguf_parameters writes phi3 fields with sliding_window=0 default."""
from auto_round.export.export_to_gguf.conversion.phi import Phi3MiniModel
obj = _make_mock_model(
Phi3MiniModel,
{
"hidden_size": 3072,
"num_attention_heads": 32,
"num_key_value_heads": 32,
"rms_norm_eps": 1e-5,
"max_position_embeddings": 4096,
"original_max_position_embeddings": 4096,
"partial_rotary_factor": 1.0,
"intermediate_size": 8192,
},
)
obj.rope_parameters = {"full_attention": {"rope_theta": 10000.0}, "original_max_position_embeddings": 4096}
obj.set_gguf_parameters()
w = obj.gguf_writer
w.add_context_length.assert_called_once_with(4096)
w.add_rope_scaling_orig_ctx_len.assert_called_once_with(4096)
w.add_embedding_length.assert_called_once_with(3072)
w.add_feed_forward_length.assert_called_once_with(8192)
w.add_head_count.assert_called_once_with(32)
w.add_head_count_kv.assert_called_once_with(32)
w.add_layer_norm_rms_eps.assert_called_once_with(1e-5)
w.add_rope_dimension_count.assert_called_once_with(96)
w.add_rope_freq_base.assert_called_once_with(10000.0)
# sliding_window defaults to 0 when not in hparams
w.add_sliding_window.assert_called_once_with(0)
def test_phi3_set_gguf_parameters_with_sliding_window(self):
"""Test Phi3MiniModel.set_gguf_parameters uses sliding_window from hparams when present."""
from auto_round.export.export_to_gguf.conversion.phi import Phi3MiniModel
obj = _make_mock_model(
Phi3MiniModel,
{
"hidden_size": 3072,
"num_attention_heads": 32,
"num_key_value_heads": 32,
"rms_norm_eps": 1e-5,
"max_position_embeddings": 4096,
"original_max_position_embeddings": 4096,
"intermediate_size": 8192,
"sliding_window": 512,
},
)
obj.rope_parameters = {"rope_theta": 10000.0, "original_max_position_embeddings": 4096}
obj.set_gguf_parameters()
obj.gguf_writer.add_sliding_window.assert_called_once_with(512)
def test_phi3_generate_extra_tensors_longrope(self):
"""Test Phi3MiniModel.generate_extra_tensors produces ROPE_FACTORS_LONG/SHORT."""
from auto_round.export.export_to_gguf.conversion.phi import Phi3MiniModel
obj = _make_mock_model(
Phi3MiniModel,
{
"hidden_size": 3072,
"num_attention_heads": 32,
"max_position_embeddings": 131072,
"original_max_position_embeddings": 4096,
"partial_rotary_factor": 1.0,
"intermediate_size": 8192,
"num_key_value_heads": 32,
"rms_norm_eps": 1e-5,
# rope_dims = int(1.0 * 3072) // 32 = 96, factors length = 48
"rope_scaling": {
"rope_type": "longrope",
"long_factor": [1.0] * 48,
"short_factor": [1.0] * 48,
},
},
)
obj.rope_parameters = {
"rope_theta": 10000.0,
"original_max_position_embeddings": 4096,
"rope_type": "longrope",
"long_factor": [1.0] * 48,
"short_factor": [1.0] * 48,
}
results = list(obj.generate_extra_tensors())
# Should yield 2 tensors: long and short factors
assert len(results) == 2
# format_tensor_name(prefix) yields a fully qualified name like "blk.0.rope_factors_long.weight"
# ensure both tensors are present and contain the expected tensor kinds
from auto_round.export.export_to_gguf.conversion.base import gguf
assert any(gguf.MODEL_TENSOR.ROPE_FACTORS_LONG.name in n for n, _ in results)
assert any(gguf.MODEL_TENSOR.ROPE_FACTORS_SHORT.name in n for n, _ in results)
def test_phimoe_set_gguf_parameters(self):
"""Test PhiMoeModel.set_gguf_parameters adds expert_count and expert_used_count."""
from auto_round.export.export_to_gguf.conversion.phi import PhiMoeModel
obj = _make_mock_model(
PhiMoeModel,
{
"hidden_size": 4096,
"num_attention_heads": 32,
"num_key_value_heads": 8,
"rms_norm_eps": 1e-5,
"max_position_embeddings": 4096,
"original_max_position_embeddings": 4096,
"intermediate_size": 14336,
"num_local_experts": 16,
"num_experts_per_tok": 2,
},
)
obj.rope_parameters = {"rope_theta": 10000.0}
with patch.object(PhiMoeModel.__mro__[1], "set_gguf_parameters", lambda self: None):
obj.set_gguf_parameters()
obj.gguf_writer.add_expert_used_count.assert_called_once_with(2)
obj.gguf_writer.add_expert_count.assert_called_once_with(16)
def test_phimoe_modify_tensors_experts_merge(self):
"""Test PhiMoeModel.modify_tensors merges block_sparse_moe.experts tensors."""
from auto_round.export.export_to_gguf.conversion.phi import PhiMoeModel
obj = _make_mock_model(
PhiMoeModel,
{
"num_local_experts": 2,
"hidden_size": 8,
"num_attention_heads": 2,
"num_key_value_heads": 2,
"rms_norm_eps": 1e-5,
"intermediate_size": 8,
},
)
captured = []
with patch.object(
PhiMoeModel.__mro__[1], "modify_tensors", lambda self, d, n, b: captured.append(n) or iter([(n, d)])
):
# 2 experts * 3 weights = 6 tensors trigger merge
for xid in range(2):
for wid in ["w1", "w2", "w3"]:
ename = f"model.layers.0.block_sparse_moe.experts.{xid}.{wid}.weight"
list(obj.modify_tensors(torch.randn(8, 8), ename, bid=0))
# 3 merged names expected
assert any("experts.w1.weight" in n for n in captured)
assert any("experts.w2.weight" in n for n in captured)
assert any("experts.w3.weight" in n for n in captured)
def test_phi4_vision_set_gguf_parameters(self):
"""Test Phi4VisionMmprojModel.set_gguf_parameters writes PHI4 projector + use_gelu."""
from auto_round.export.export_to_gguf.conversion.base import gguf
from auto_round.export.export_to_gguf.conversion.phi import Phi4VisionMmprojModel
obj = _make_mock_model(
Phi4VisionMmprojModel,
{
"layer_norm_eps": 1e-6,
"num_hidden_layers": 24,
},
)
obj.min_pixels = 256 * 16 * 16
obj.max_pixels = 1280 * 16 * 16
# vision-related fields required for the parent class paths
obj.hparams_vision = {"layer_norm_eps": 1e-6, "num_hidden_layers": 24}
with patch.object(Phi4VisionMmprojModel.__mro__[1], "set_gguf_parameters", lambda self: None):
obj.set_gguf_parameters()
w = obj.gguf_writer
w.add_clip_projector_type.assert_called_once_with(gguf.VisionProjectorType.PHI4)
w.add_vision_use_gelu.assert_called_once_with(True)
w.add_vision_min_pixels.assert_called_once_with(256 * 16 * 16)
w.add_vision_max_pixels.assert_called_once_with(1280 * 16 * 16)
def test_phi4_vision_filter_tensors(self):
"""Test Phi4VisionMmprojModel.filter_tensors handles vision_tower/mm_projector paths."""
from auto_round.export.export_to_gguf.conversion.phi import Phi4VisionMmprojModel
# Vision tower non-prefixed -> None
assert Phi4VisionMmprojModel.filter_tensors(("model.embed_tokens.weight", lambda: None)) is None
# vision_tower.* tensors are kept (returned)
result = Phi4VisionMmprojModel.filter_tensors(("vision_tower.encoder.layers.0.ln.weight", lambda: None))
assert result is not None
# Drop post_layernorm and vision_model.head
assert (
Phi4VisionMmprojModel.filter_tensors(("vision_tower.vision_model.post_layernorm.weight", lambda: None))
is None
)
assert Phi4VisionMmprojModel.filter_tensors(("vision_tower.vision_model.head.weight", lambda: None)) is None
def test_phi4_vision_modify_tensors_mm_projector(self):
"""Test Phi4VisionMmprojModel.modify_tensors maps mm_projector.0./2. to V_MMPROJ."""
from auto_round.export.export_to_gguf.conversion.base import gguf
from auto_round.export.export_to_gguf.conversion.phi import Phi4VisionMmprojModel
obj = _make_mock_model(Phi4VisionMmprojModel, {})
# Test mm_projector.0 mapping (becomes V_MMPROJ.0.weight)
data = torch.randn(8, 8)
result = list(obj.modify_tensors(data, "model.mm_projector.0.weight", bid=None))
assert len(result) == 1
assert result[0][0] == obj.format_tensor_name(gguf.MODEL_TENSOR.V_MMPROJ, 0, suffix=".weight")
# mm_projector.2.bias
result_bias = list(obj.modify_tensors(data, "model.mm_projector.2.bias", bid=None))
assert len(result_bias) == 1
assert ".bias" in result_bias[0][0]
# mm_projector.1 (Linear 1, FC1) should be dropped
result_drop = list(obj.modify_tensors(data, "model.mm_projector.1.weight", bid=None))
assert result_drop == []
def test_phi4_vision_modify_tensors_patch_embedding_reshape(self):
"""Test Phi4VisionMmprojModel reshapes 2D patch_embedding to (out, c, p, p)."""
from auto_round.export.export_to_gguf.conversion.phi import Phi4VisionMmprojModel
obj = _make_mock_model(Phi4VisionMmprojModel, {})
obj.vision_last_layer_idx = 99 # so we don't drop on bid
# patch_area = 4 (patch_size=2), so input dim 12 -> 3 channels
# data shape [out_dim, in_dim=12] -> reshape to [out_dim, 2, 2, 3] -> permute [out_dim, 3, 2, 2]
data = torch.randn(8, 12)
obj.hparams_vision = {"patch_size": 2}
with patch.object(Phi4VisionMmprojModel.__mro__[1], "modify_tensors", lambda self, d, n, b: iter([(n, d)])):
result = list(
obj.modify_tensors(data, "vision_tower.vision_model.embeddings.patch_embedding.weight", bid=0)
)
# After view + permute: shape should be (8, 3, 2, 2)
assert result[0][1].shape == (8, 3, 2, 2)
# ==============================================================================
# qwen.py tests
# ==============================================================================
class TestQwenConversion:
"""Tests for Qwen / Qwen2 / Qwen2MoE conversion module."""
def test_qwen_bpe_merges_pairs(self):
"""Test QwenModel.bpe merges the highest-rank adjacent bytes."""
from auto_round.export.export_to_gguf.conversion.qwen import QwenModel
# Pair ranks: ('a','b') -> 0, ('b','c') -> 1, ('c','d') -> 2
ranks = {b"ab": 0, b"bc": 1, b"cd": 2}
# Token = "abcd", parts initially = [b'a', b'b', b'c', b'd']
# Step 1: min pair = (a,b) rank 0 -> merge to b'ab'
# parts = [b'ab', b'c', b'd']
# Step 2: no more pairs in ranks
parts = QwenModel.bpe(ranks, b"abcd")
assert parts == [b"ab", b"cd"]
def test_qwen_bpe_respects_max_rank(self):
"""Test QwenModel.bpe stops merging when min rank >= max_rank."""
from auto_round.export.export_to_gguf.conversion.qwen import QwenModel
ranks = {b"ab": 0}
# max_rank=0 -> merge stops immediately at rank 0
parts = QwenModel.bpe(ranks, b"abcd", max_rank=0)
assert parts == [b"a", b"b", b"c", b"d"]
def test_qwen_bpe_no_matchable_pairs(self):
"""Test QwenModel.bpe returns parts unchanged when no pair is mergeable."""
from auto_round.export.export_to_gguf.conversion.qwen import QwenModel
ranks = {b"xx": 0} # no matching pairs in "abcd"
parts = QwenModel.bpe(ranks, b"abcd")
assert parts == [b"a", b"b", b"c", b"d"]
def test_qwen2_modify_tensors_qwen2model_prefix(self):
"""Test Qwen2Model.modify_tensors adds model. prefix when hf_arch=Qwen2Model."""
from auto_round.export.export_to_gguf.conversion.qwen import Qwen2Model
obj = _make_mock_model(Qwen2Model, {})
obj.hf_arch = "Qwen2Model"
data = torch.randn(8, 8)
captured = []
with patch.object(
Qwen2Model.__mro__[1], "modify_tensors", lambda self, d, n, b: captured.append(n) or iter([(n, d)])
):
list(obj.modify_tensors(data, "embed_tokens.weight", bid=None))
# Name should be prefixed with "model."
assert captured[0] == "model.embed_tokens.weight"
def test_qwen2_modify_tensors_for_causal_lm_passthrough(self):
"""Test Qwen2Model.modify_tensors does NOT prefix when hf_arch=Qwen2ForCausalLM."""
from auto_round.export.export_to_gguf.conversion.qwen import Qwen2Model
obj = _make_mock_model(Qwen2Model, {})
obj.hf_arch = "Qwen2ForCausalLM"
data = torch.randn(8, 8)
captured = []
with patch.object(
Qwen2Model.__mro__[1], "modify_tensors", lambda self, d, n, b: captured.append(n) or iter([(n, d)])
):
list(obj.modify_tensors(data, "model.embed_tokens.weight", bid=None))
# Name stays unchanged when hf_arch != "Qwen2Model"
assert captured[0] == "model.embed_tokens.weight"
def test_qwen2moe_set_gguf_parameters(self):
"""Test Qwen2MoeModel.set_gguf_parameters writes expert FF / shared FF lengths."""
from auto_round.export.export_to_gguf.conversion.qwen import Qwen2MoeModel
obj = _make_mock_model(
Qwen2MoeModel,
{
"moe_intermediate_size": 1536,
"shared_expert_intermediate_size": 512,
},
)
with patch.object(Qwen2MoeModel.__mro__[1], "set_gguf_parameters", lambda self: None):
obj.set_gguf_parameters()
w = obj.gguf_writer
w.add_expert_feed_forward_length.assert_called_once_with(1536)
w.add_expert_shared_feed_forward_length.assert_called_once_with(512)
def test_qwen2moe_set_gguf_parameters_no_shared(self):
"""Test Qwen2MoeModel.set_gguf_parameters when shared_expert_intermediate_size is missing."""
from auto_round.export.export_to_gguf.conversion.qwen import Qwen2MoeModel
obj = _make_mock_model(Qwen2MoeModel, {"moe_intermediate_size": 1536})
with patch.object(Qwen2MoeModel.__mro__[1], "set_gguf_parameters", lambda self: None):
obj.set_gguf_parameters()
obj.gguf_writer.add_expert_feed_forward_length.assert_called_once_with(1536)
obj.gguf_writer.add_expert_shared_feed_forward_length.assert_not_called()
def test_qwen2moe_modify_tensors_gate_up_split(self):
"""Test Qwen2MoeModel.modify_tensors splits mlp.experts.gate_up_proj into gate/up."""
from auto_round.export.export_to_gguf.conversion.qwen import Qwen2MoeModel
obj = _make_mock_model(Qwen2MoeModel, {"num_local_experts": 2})
captured = []
# [n_expert, 2*n_ff, n_embd] = [2, 8, 4] -> gate shape [2, 4, 4], up shape [2, 4, 4]
data = torch.randn(2, 8, 4)
with patch.object(
Qwen2MoeModel.__mro__[1], "modify_tensors", lambda self, d, n, b: captured.append((n, d)) or iter([(n, d)])
):
list(obj.modify_tensors(data, "model.layers.0.mlp.experts.gate_up_proj.weight", bid=0))
# Should yield 2 outputs: gate_proj and up_proj
assert len(captured) == 2
assert any("gate_proj.weight" in n for n, _ in captured)
assert any("up_proj.weight" in n for n, _ in captured)
def test_qwen2moe_modify_tensors_gate_up_invalid_shape(self):
"""Test Qwen2MoeModel.modify_tensors raises on invalid gate_up_proj shape."""
from auto_round.export.export_to_gguf.conversion.qwen import Qwen2MoeModel
obj = _make_mock_model(Qwen2MoeModel, {})
# ndim < 3 or shape[-2] % 2 != 0 -> ValueError
data = torch.randn(2, 3, 4) # shape[-2]=3 is odd
with pytest.raises(ValueError, match="gate_up_proj"):
list(obj.modify_tensors(data, "model.layers.0.mlp.experts.gate_up_proj.weight", bid=0))
def test_qwen2moe_modify_tensors_down_proj(self):
"""Test Qwen2MoeModel.modify_tensors handles mlp.experts.down_proj passthrough."""
from auto_round.export.export_to_gguf.conversion.qwen import Qwen2MoeModel
obj = _make_mock_model(Qwen2MoeModel, {})
data = torch.randn(2, 4, 8)
captured = []
with patch.object(
Qwen2MoeModel.__mro__[1], "modify_tensors", lambda self, d, n, b: captured.append(n) or iter([(n, d)])
):
list(obj.modify_tensors(data, "model.layers.0.mlp.experts.down_proj.weight", bid=0))
assert len(captured) == 1
assert "down_proj.weight" in captured[0]
# ==============================================================================
# qwen3vl.py tests
# ==============================================================================
class TestQwen3VlConversion:
"""Tests for Qwen3VL vision conversion module."""
def test_set_gguf_parameters(self):
"""Test Qwen3VLVisionModel.set_gguf_parameters writes vision projector fields."""
from auto_round.export.export_to_gguf.conversion.base import gguf
from auto_round.export.export_to_gguf.conversion.qwen3vl import Qwen3VLVisionModel
obj = _make_mock_model(Qwen3VLVisionModel)
# has_audio_encoder is required on the object
obj.has_audio_encoder = False
# __init__ populates num_attention_heads/num_hidden_layers + is_deepstack_layers
# from these keys. image_size is computed from num_position_embeddings.
obj.hparams_vision = {
"spatial_merge_size": 2,
"patch_size": 16,
"num_heads": 16,
"depth": 24,
"num_position_embeddings": 2304,
"deepstack_visual_indexes": [],
}
# Manually run the __init__ logic that populates is_deepstack_layers
if "num_attention_heads" not in obj.hparams_vision:
obj.hparams_vision["num_attention_heads"] = obj.hparams_vision.get("num_heads")
if "num_hidden_layers" not in obj.hparams_vision:
obj.hparams_vision["num_hidden_layers"] = obj.hparams_vision.get("depth")
obj.is_deepstack_layers = [False] * int(obj.hparams_vision["num_hidden_layers"] or 0)
for idx in obj.hparams_vision.get("deepstack_visual_indexes", []):
obj.is_deepstack_layers[idx] = True
with patch.object(Qwen3VLVisionModel.__mro__[1], "set_gguf_parameters", lambda self: None):
obj.set_gguf_parameters()
w = obj.gguf_writer
w.add_clip_projector_type.assert_called_once_with(gguf.VisionProjectorType.QWEN3VL)
w.add_vision_use_gelu.assert_called_once_with(True)
w.add_vision_spatial_merge_size.assert_called_once_with(2)
# ==============================================================================
# ernie.py tests
# ==============================================================================
class TestErnieConversion:
"""Tests for Ernie / Ernie4_5 / PaddleOCR conversion module."""
def test_ernie4_5_filter_tensors_renames_ernie_prefix(self):
"""Test Ernie4_5Model.filter_tensors renames 'ernie.' prefix to 'model.'."""
from auto_round.export.export_to_gguf.conversion.ernie import Ernie4_5Model
def parent_filter(item):
return item
with patch.object(Ernie4_5Model.__mro__[1], "filter_tensors", staticmethod(parent_filter)):
# When name starts with 'ernie.', replace with 'model.'
result = Ernie4_5Model.filter_tensors(("ernie.embed_tokens.weight", lambda: None))
assert result[0] == "model.embed_tokens.weight"
def test_ernie4_5_moe_set_gguf_parameters(self):
"""Test Ernie4_5MoeModel.set_gguf_parameters writes MoE-specific fields."""
from auto_round.export.export_to_gguf.conversion.ernie import Ernie4_5MoeModel
obj = _make_mock_model(
Ernie4_5MoeModel,
{
"moe_num_experts": 8,
"moe_k": 2,
"moe_layer_interval": 2,
"moe_layer_start_index": 1,
"moe_intermediate_size": 1536,
"moe_num_shared_experts": 2,
"intermediate_size": 12288,
"num_key_value_heads": 8,
"max_position_embeddings": 131072,
"hidden_size": 4096,
"num_attention_heads": 32,
},
)
# __init__ sets _experts
obj._experts = [{} for _ in range(obj.block_count)]
with patch.object(Ernie4_5MoeModel.__mro__[1], "set_gguf_parameters", lambda self: None):
obj.set_gguf_parameters()
w = obj.gguf_writer
w.add_expert_count.assert_called_once_with(8)
w.add_expert_used_count.assert_called_once_with(2)
w.add_interleave_moe_layer_step.assert_called_once_with(2)
w.add_leading_dense_block_count.assert_called_once_with(1)
w.add_expert_feed_forward_length.assert_called_once_with(1536)
w.add_expert_shared_count.assert_called_once_with(2)
# shared FF length = intermediate_size // num_key_value_heads = 12288 // 8 = 1536
w.add_expert_shared_feed_forward_length.assert_called_once_with(1536)
def test_ernie4_5_moe_filter_drops_mtp(self):
"""Test Ernie4_5MoeModel.filter_tensors drops MTP tensors."""
from auto_round.export.export_to_gguf.conversion.ernie import Ernie4_5MoeModel
# All MTP-related prefixes should be filtered out
for name in [
"model.mtp_block.0.weight",
"model.mtp_emb_norm.3.weight",
"model.mtp_hidden_norm.5.weight",
"model.mtp_linear_proj.2.weight",
]:
assert Ernie4_5MoeModel.filter_tensors((name, lambda: None)) is None
def test_paddleocr_vision_set_gguf_parameters(self):
"""Test PaddleOCRVisionModel.set_gguf_parameters writes vision projector fields."""
from auto_round.export.export_to_gguf.conversion.base import gguf
from auto_round.export.export_to_gguf.conversion.ernie import PaddleOCRVisionModel
obj = _make_mock_model(PaddleOCRVisionModel)
obj.hparams_vision = {"rms_norm_eps": 1e-6}
obj.preprocessor_config = {"min_pixels": 256, "max_pixels": 1024}
obj.min_pixels = 256
obj.max_pixels = 1024
obj.hparams_vision["image_size"] = 32 # int(sqrt(1024))
with patch.object(PaddleOCRVisionModel.__mro__[1], "set_gguf_parameters", lambda self: None):
obj.set_gguf_parameters()
w = obj.gguf_writer
w.add_clip_projector_type.assert_called_once_with(gguf.VisionProjectorType.PADDLEOCR)
w.add_vision_max_pixels.assert_called_once_with(1024)
w.add_vision_min_pixels.assert_called_once_with(256)
w.add_vision_use_gelu.assert_called_once_with(True)
w.add_vision_attention_layernorm_eps.assert_called_once_with(1e-6)
def test_paddleocr_vision_filter_drops_non_vision(self):
"""Test PaddleOCRVisionModel.filter_tensors drops non-vision / non-mlp_AR tensors."""
from auto_round.export.export_to_gguf.conversion.ernie import PaddleOCRVisionModel
# tensors without 'vision_model' or 'mlp_AR' should be dropped
assert PaddleOCRVisionModel.filter_tensors(("lm_head.weight", lambda: None)) is None
# packing_position_embedding and vision_model.head are dropped even for vision
assert (
PaddleOCRVisionModel.filter_tensors(("vision_model.packing_position_embedding.weight", lambda: None))
is None
)
assert PaddleOCRVisionModel.filter_tensors(("vision_model.head.weight", lambda: None)) is None
def test_ernie4_5_modify_tensors_qkv_split(self):
"""Test Ernie4_5Model.modify_tensors splits qkv_proj into q/k/v."""
from auto_round.export.export_to_gguf.conversion.ernie import Ernie4_5Model
obj = _make_mock_model(
Ernie4_5Model,
{
"num_attention_heads": 32,
"num_key_value_heads": 8,
"head_dim": 128,
"hidden_size": 4096,
},
)
captured = []
with patch.object(
Ernie4_5Model.__mro__[1], "modify_tensors", lambda self, d, n, b: captured.append(n) or iter([(n, d)])
):
data = torch.randn((32 + 2 * 8) * 128, 4096) # total_qkv dim
list(obj.modify_tensors(data, "model.layers.0.self_attn.qkv_proj.weight", bid=0))
# 3 outputs: q_proj, k_proj, v_proj
assert len(captured) == 3
assert any("q_proj.weight" in n for n in captured)
assert any("k_proj.weight" in n for n in captured)
assert any("v_proj.weight" in n for n in captured)
def test_ernie4_5_modify_tensors_up_gate_split(self):
"""Test Ernie4_5Model.modify_tensors splits up_gate_proj into gate/up."""
from auto_round.export.export_to_gguf.conversion.ernie import Ernie4_5Model
obj = _make_mock_model(
Ernie4_5Model,
{
"num_attention_heads": 32,
"num_key_value_heads": 8,
"head_dim": 128,
"hidden_size": 4096,
},
)
captured = []
with patch.object(
Ernie4_5Model.__mro__[1], "modify_tensors", lambda self, d, n, b: captured.append(n) or iter([(n, d)])
):
data = torch.randn(8192, 4096) # 2 * intermediate_size
list(obj.modify_tensors(data, "model.layers.0.mlp.up_gate_proj.weight", bid=0))
# 2 outputs: gate_proj, up_proj
assert len(captured) == 2
assert any("gate_proj.weight" in n for n in captured)
assert any("up_proj.weight" in n for n in captured)
def test_ernie4_5_modify_tensors_passthrough(self):
"""Test Ernie4_5Model.modify_tensors passes through non-qkv/up_gate tensors."""
from auto_round.export.export_to_gguf.conversion.ernie import Ernie4_5Model
obj = _make_mock_model(
Ernie4_5Model,
{
"num_attention_heads": 32,
"num_key_value_heads": 8,
"head_dim": 128,
"hidden_size": 4096,
},
)
captured = []
with patch.object(
Ernie4_5Model.__mro__[1], "modify_tensors", lambda self, d, n, b: captured.append(n) or iter([(n, d)])
):
data = torch.randn(4096, 4096)
list(obj.modify_tensors(data, "model.embed_tokens.weight", bid=None))
assert captured == ["model.embed_tokens.weight"]
def test_ernie4_5_moe_modify_tensors_expert_merge(self):
"""Test Ernie4_5MoeModel.modify_tensors merges mlp.experts tensors."""
from auto_round.export.export_to_gguf.conversion.ernie import Ernie4_5MoeModel
obj = _make_mock_model(
Ernie4_5MoeModel,
{
"moe_num_experts": 2,
"num_attention_heads": 32,
"num_key_value_heads": 8,
"head_dim": 128,
"hidden_size": 4096,
},
)
# __init__ sets _experts
obj._experts = [{} for _ in range(obj.block_count)]
captured = []
with patch.object(
Ernie4_5MoeModel.__mro__[2], "modify_tensors", lambda self, d, n, b: captured.append(n) or iter([(n, d)])
):
# Feed 3 experts * 3 weights = 6 tensors
for xid in range(2):
for wid in ["gate_proj", "up_proj", "down_proj"]:
ename = f"model.layers.0.mlp.experts.{xid}.{wid}.weight"
list(obj.modify_tensors(torch.randn(1024, 1024), ename, bid=0))
# 3 merged names expected
assert any("experts.gate_proj.weight" in n for n in captured)
assert any("experts.up_proj.weight" in n for n in captured)
assert any("experts.down_proj.weight" in n for n in captured)
# ==============================================================================
# mistral.py tests
# ==============================================================================
class TestMistralConversion:
"""Tests for Mistral conversion module (static methods and helpers)."""
def test_dequant_model_sets_fp8_quantization_config(self):
"""Test MistralModel.dequant_model converts qformat_weight=fp8_e4m3 to fp8 config."""
from auto_round.export.export_to_gguf.conversion.mistral import MistralModel
obj = _make_mock_model(MistralModel)
obj.hparams["quantization"] = {"qformat_weight": "fp8_e4m3"}
# Avoid actually running super().dequant_model() which would access safetensors files
with patch.object(MistralModel.__mro__[1], "dequant_model", lambda self: None):
obj.dequant_model()
assert obj.hparams["quantization_config"]["quant_method"] == "fp8"
assert obj.hparams["quantization_config"]["activation_scheme"] == "static"
def test_set_mistral_config_with_yarn(self):
"""Test MistralModel.set_mistral_config writes yarn rope params."""
from auto_round.export.export_to_gguf.conversion.base import gguf
from auto_round.export.export_to_gguf.conversion.mistral import MistralModel
hparams = {
"yarn": {
"apply_scale": True,
"factor": 16.0,
"beta": 32.0,
"alpha": 1.0,
"original_max_position_embeddings": 32768,
}
}
gguf_writer = MagicMock()
MistralModel.set_mistral_config(gguf_writer, hparams)
gguf_writer.add_rope_scaling_type.assert_called_once_with(gguf.RopeScalingType.YARN)
gguf_writer.add_rope_scaling_factor.assert_called_once_with(16.0)
# mscale_all_dim = 0.0 when apply_scale is True
gguf_writer.add_rope_scaling_yarn_log_mul.assert_called_once_with(0.0)
def test_set_mistral_config_with_llama_4_scaling(self):
"""Test MistralModel.set_mistral_config writes attn_temperature_scale."""
from auto_round.export.export_to_gguf.conversion.mistral import MistralModel
hparams = {"llama_4_scaling": {"beta": 0.5}}
gguf_writer = MagicMock()
MistralModel.set_mistral_config(gguf_writer, hparams)
gguf_writer.add_attn_temperature_scale.assert_called_once_with(0.5)
def test_mistral_filter_tensors_renames_expert_tensors(self):
"""Test MistralMoeModel.filter_tensors renames w1/w2/w3 to gate/down/up."""
from auto_round.export.export_to_gguf.conversion.mistral import MistralMoeModel
def parent_filter(item):
return item
with patch.object(MistralMoeModel.__mro__[1], "filter_tensors", staticmethod(parent_filter)):
# w1 -> gate_proj, w2 -> down_proj, w3 -> up_proj, plus experts -> mlp.experts
result = MistralMoeModel.filter_tensors(("model.experts.0.w1.weight", lambda: None))
assert result[0] == "model.model.mlp.experts.0.gate_proj.weight"
# ==============================================================================
# hunyuan.py tests
# ==============================================================================
class TestHunyuanConversion:
"""Tests for HunYuan conversion module."""
def test_hunyuan_moe_set_gguf_parameters(self):
"""Test HunYuanMoEModel.set_gguf_parameters writes expert shared FF length and top-k."""
from auto_round.export.export_to_gguf.conversion.hunyuan import HunYuanMoEModel
obj = _make_mock_model(
HunYuanMoEModel,
{
"intermediate_size": 12288,
"moe_intermediate_size": [1536, 1536, 1536],
"moe_topk": [8, 8, 8],
"num_shared_expert": [1, 1, 1],
"hidden_act": "silu",
"max_position_embeddings": 262144,
"hidden_size": 4096,
"num_attention_heads": 32,
},
)
with patch.object(HunYuanMoEModel.__mro__[1], "set_gguf_parameters", lambda self: None):
obj.set_gguf_parameters()
w = obj.gguf_writer
w.add_expert_shared_feed_forward_length.assert_called_once_with(12288)
w.add_expert_feed_forward_length.assert_called_once_with(1536)
w.add_expert_used_count.assert_called_once_with(8)
w.add_expert_shared_count.assert_called_once_with(1)
def test_hunyuan_dense_get_eod_token_id(self):
"""Test HunYuanModel._get_eod_token_id reads from hparams."""
from auto_round.export.export_to_gguf.conversion.hunyuan import HunYuanModel
obj = _make_mock_model(HunYuanModel, {"eod_token_id": 120000})
assert obj._get_eod_token_id() == 120000
def test_hunyuan_vl_vision_set_gguf_parameters(self):
"""Test HunyuanVLVisionModel.set_gguf_parameters writes vision projector fields."""
from auto_round.export.export_to_gguf.conversion.base import gguf
from auto_round.export.export_to_gguf.conversion.hunyuan import HunyuanVLVisionModel
obj = _make_mock_model(HunyuanVLVisionModel)
obj.hparams_vision = {
"rms_norm_eps": 1e-5,
"spatial_merge_size": 2,
"max_image_size": 2048,
}
obj.preprocessor_config = {"min_pixels": 256, "max_pixels": 1024}
with patch.object(HunyuanVLVisionModel.__mro__[1], "set_gguf_parameters", lambda self: None):
obj.set_gguf_parameters()
w = obj.gguf_writer
w.add_clip_projector_type.assert_called_once_with(gguf.VisionProjectorType.HUNYUANVL)
w.add_vision_use_gelu.assert_called_once_with(True)
w.add_vision_attention_layernorm_eps.assert_called_once_with(1e-5)
w.add_vision_spatial_merge_size.assert_called_once_with(2)
w.add_vision_min_pixels.assert_called_once_with(256)
w.add_vision_max_pixels.assert_called_once_with(1024)
def test_hunyuan_vl_vision_filter_drops_non_vit(self):
"""Test HunyuanVLVisionModel.filter_tensors drops non-vit tensors."""
from auto_round.export.export_to_gguf.conversion.hunyuan import HunyuanVLVisionModel
# tensors not starting with 'vit.' should be dropped
assert HunyuanVLVisionModel.filter_tensors(("model.layers.0.weight", lambda: None)) is None
assert HunyuanVLVisionModel.filter_tensors(("language_model.weight", lambda: None)) is None
def test_hunyuan_vl_text_set_gguf_parameters(self):
"""Test HunyuanVLTextModel.set_gguf_parameters writes xdrope fields when applicable."""
from auto_round.export.export_to_gguf.conversion.hunyuan import HunyuanVLTextModel
obj = _make_mock_model(
HunyuanVLTextModel,
{
"rope_type": "xdrope",
"rope_theta": 10000.0,
"alpha": 1000,
"hidden_size": 4096,
"num_attention_heads": 32,
"max_position_embeddings": 32768,
"intermediate_size": 12288,
},
)
obj.rope_parameters = {
"rope_type": "xdrope",
"rope_theta": 10000.0,
"alpha": 1000,
"xdrope_section": [16, 16, 16, 16],
}
with patch.object(HunyuanVLTextModel.__mro__[1], "set_gguf_parameters", lambda self: None):
obj.set_gguf_parameters()
obj.gguf_writer.add_rope_freq_base.assert_called_once_with(10000.0)
obj.gguf_writer.add_rope_scaling_alpha.assert_called_once_with(1000.0)
def test_hunyuan_moe_modify_tensors_expert_merge(self):
"""Test HunYuanMoEModel.modify_tensors merges mlp.experts tensors."""
from auto_round.export.export_to_gguf.conversion.hunyuan import HunYuanMoEModel
obj = _make_mock_model(
HunYuanMoEModel,
{
"num_local_experts": 2,
},
)
captured = []
with patch.object(
HunYuanMoEModel.__mro__[1], "modify_tensors", lambda self, d, n, b: captured.append(n) or iter([(n, d)])
):
# 3 experts * 3 weights = 6 tensors trigger merge
for xid in range(2):
for wid in ["down_proj", "gate_proj", "up_proj"]:
ename = f"model.layers.0.mlp.experts.{xid}.{wid}.weight"
list(obj.modify_tensors(torch.randn(8, 8), ename, bid=0))
assert any("experts.down_proj.weight" in n for n in captured)
assert any("experts.gate_proj.weight" in n for n in captured)
assert any("experts.up_proj.weight" in n for n in captured)
def test_hunyuan_moe_modify_tensors_drops_tied_lm_head(self):
"""Test HunYuanMoEModel.modify_tensors drops lm_head when tied."""
from auto_round.export.export_to_gguf.conversion.hunyuan import HunYuanMoEModel
obj = _make_mock_model(HunYuanMoEModel, {"tie_word_embeddings": True})
result = list(obj.modify_tensors(torch.zeros(8), "lm_head.weight", bid=None))
assert result == []
def test_hunyuan_dense_modify_tensors_passthrough(self):
"""Test HunYuanModel.modify_tensors passes through non-expert tensors."""
from auto_round.export.export_to_gguf.conversion.hunyuan import HunYuanModel
obj = _make_mock_model(HunYuanModel)
with patch.object(HunYuanModel.__mro__[1], "modify_tensors", lambda self, d, n, b: iter([(n, d)])):
data = torch.randn(8, 8)
result = list(obj.modify_tensors(data, "model.layers.0.self_attn.q_proj.weight", bid=0))
assert torch.equal(result[0][1], data)
def test_hunyuan_vl_vision_modify_tensors_strips_cls_position(self):
"""Test HunyuanVLVisionModel.modify_tensors strips CLS row from position_embedding."""
from auto_round.export.export_to_gguf.conversion.hunyuan import HunyuanVLVisionModel
obj = _make_mock_model(HunyuanVLVisionModel)
# position_embedding: [n_patches+1, n_embd] -> [n_patches, n_embd]
data = torch.arange(2 * 16).reshape(2, 16).float()
with patch.object(HunyuanVLVisionModel.__mro__[1], "modify_tensors", lambda self, d, n, b: iter([(n, d)])):
result = list(obj.modify_tensors(data, "vit.position_embedding.weight", bid=0))
# First row stripped -> result has shape [1, 16]
assert result[0][1].shape == (1, 16)
# ==============================================================================
# bert.py tests
# ==============================================================================
class TestBertConversion:
"""Tests for Bert conversion module."""
def test_bert_init_drops_dummy_labels(self):
"""Test BertModel.__init__ drops dummy 'LABEL_0'-style id2label entries."""
from auto_round.export.export_to_gguf.conversion.bert import BertModel
# When id2label has only "LABEL_0", "LABEL_1", it should be cleared to None
obj = _make_mock_model(BertModel, {"id2label": {0: "LABEL_0", 1: "LABEL_1"}})
# Manually run the init logic for cls_out_labels
cls_out_labels = obj.hparams.get("id2label")
if len(cls_out_labels) == 2 and cls_out_labels[0] == "LABEL_0":
cls_out_labels = None
obj.cls_out_labels = cls_out_labels
assert obj.cls_out_labels is None
def test_bert_init_keeps_real_labels(self):
"""Test BertModel.__init__ keeps real id2label mapping."""
from auto_round.export.export_to_gguf.conversion.bert import BertModel
obj = _make_mock_model(BertModel, {"id2label": {0: "positive", 1: "negative"}})
# Manually run the init logic for cls_out_labels
cls_out_labels = obj.hparams.get("id2label")
if len(cls_out_labels) == 2 and cls_out_labels[0] == "LABEL_0":
cls_out_labels = None
obj.cls_out_labels = cls_out_labels
assert obj.cls_out_labels == {0: "positive", 1: "negative"}
def test_bert_set_gguf_parameters(self):
"""Test BertModel.set_gguf_parameters writes non-causal flag."""
from auto_round.export.export_to_gguf.conversion.bert import BertModel
obj = _make_mock_model(BertModel)
obj.cls_out_labels = None
with (
patch.object(BertModel.__mro__[1], "set_gguf_parameters", lambda self: None),
patch.object(obj, "_try_set_pooling_type"),
):
obj.set_gguf_parameters()
obj.gguf_writer.add_causal_attention.assert_called_once_with(False)
def test_bert_set_gguf_parameters_with_classifier_labels(self):
"""Test BertModel.set_gguf_parameters adds classifier labels when present."""
from auto_round.export.export_to_gguf.conversion.bert import BertModel
obj = _make_mock_model(BertModel)
obj.cls_out_labels = {"0": "NEGATIVE", "1": "POSITIVE"}
with (
patch.object(BertModel.__mro__[1], "set_gguf_parameters", lambda self: None),
patch.object(obj, "_try_set_pooling_type"),
):
obj.set_gguf_parameters()
obj.gguf_writer.add_classifier_output_labels.assert_called_once_with(["NEGATIVE", "POSITIVE"])
def test_bert_filter_tensors_strips_bert_prefix(self):
"""Test BertModel.filter_tensors strips leading 'bert.' prefix."""
from auto_round.export.export_to_gguf.conversion.bert import BertModel
def parent_filter(item):
return item
with patch.object(BertModel.__mro__[1], "filter_tensors", staticmethod(parent_filter)):
result = BertModel.filter_tensors(("bert.embeddings.weight", lambda: None))
assert result[0] == "embeddings.weight"
def test_bert_filter_tensors_renames_gamma_beta(self):
"""Test BertModel.filter_tensors converts .gamma -> .weight and .beta -> .bias."""
from auto_round.export.export_to_gguf.conversion.bert import BertModel
def parent_filter(item):
return item
with patch.object(BertModel.__mro__[1], "filter_tensors", staticmethod(parent_filter)):
result = BertModel.filter_tensors(("encoder.layer.0.attention.self.LayerNorm.gamma", lambda: None))
assert result[0].endswith(".weight")
result = BertModel.filter_tensors(("encoder.layer.0.attention.self.LayerNorm.beta", lambda: None))
assert result[0].endswith(".bias")
def test_bert_filter_tensors_drops_position_ids_pooler_cls(self):
"""Test BertModel.filter_tensors drops position_ids, pooler, cls.predictions, cls.seq_relationship."""
from auto_round.export.export_to_gguf.conversion.bert import BertModel
for name in [
"embeddings.position_ids",
"pooler.dense.weight",
"pooler.dense.bias",
"cls.predictions.decoder.weight",
"cls.seq_relationship.weight",
]:
assert BertModel.filter_tensors((name, lambda: None)) is None
def test_bert_modify_tensors_classifier_rename(self):
"""Test BertModel.modify_tensors renames classifier.weight/bias when cls_out_labels present."""
from auto_round.export.export_to_gguf.conversion.bert import BertModel
obj = _make_mock_model(BertModel)
obj.cls_out_labels = {"0": "NEGATIVE", "1": "POSITIVE"}
# Patch super().modify_tensors to capture renamed name
captured = []
def fake_modify(self, data, name, bid):
captured.append(name)
yield (name, data)
with patch.object(BertModel.__mro__[1], "modify_tensors", fake_modify):
data = torch.zeros(2, 768)
list(obj.modify_tensors(data, "classifier.weight", None))
list(obj.modify_tensors(data, "classifier.bias", None))
assert "classifier.out_proj.weight" in captured
assert "classifier.out_proj.bias" in captured
def test_bert_modify_tensors_no_classifier_labels(self):
"""Test BertModel.modify_tensors leaves classifier.* unchanged when no labels."""
from auto_round.export.export_to_gguf.conversion.bert import BertModel
obj = _make_mock_model(BertModel)
obj.cls_out_labels = None
captured = []
def fake_modify(self, data, name, bid):
captured.append(name)
yield (name, data)
with patch.object(BertModel.__mro__[1], "modify_tensors", fake_modify):
data = torch.zeros(2, 768)
list(obj.modify_tensors(data, "classifier.weight", None))
assert captured == ["classifier.weight"]