Skip to content

PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度

fix gguf format mtp bug (#2297)

Heng Guo947ce102c2f3
diff --git a/auto_round/envs.py b/auto_round/envs.py
index bef41f43d..a30d016e3 100644
--- a/auto_round/envs.py
+++ b/auto_round/envs.py
@@ -23,6 +23,7 @@ if TYPE_CHECKING:
AR_MODEL_FREE_SHARD_PARALLELISM: Optional[int] = None
AUTO_ROUND_CACHE: Optional[str] = None
AUTO_ROUND_GGUF_AUTO_UPDATE: bool = False
+ AR_DISABLE_GGUF_MTP_EXPORT: bool = False
LLAMA_CPP_ROOT: Optional[str] = None
AR_AUTO_SCHEME_NSAMPLES: Optional[int] = None
AR_AUTO_SCHEME_BATCH_SIZE: Optional[int] = None
@@ -90,6 +91,8 @@ environment_variables: dict[str, Callable[[], Any]] = {
"AUTO_ROUND_CACHE": lambda: os.getenv("AUTO_ROUND_CACHE", None),
"AUTO_ROUND_GGUF_AUTO_UPDATE": lambda: os.getenv("AUTO_ROUND_GGUF_AUTO_UPDATE", "0").lower()
in ("1", "true", "yes", "on"),
+ "AR_DISABLE_GGUF_MTP_EXPORT": lambda: os.getenv("AR_DISABLE_GGUF_MTP_EXPORT", "0").lower()
+ in ("1", "true", "yes", "on"),
"LLAMA_CPP_ROOT": lambda: os.getenv("LLAMA_CPP_ROOT", None),
# Controls the default number of calibration samples used by AutoScheme scoring
# when ``AutoScheme.nsamples`` is not explicitly set.
diff --git a/auto_round/export/export_to_gguf/export.py b/auto_round/export/export_to_gguf/export.py
index 476dd1f83..443e8bc0c 100644
--- a/auto_round/export/export_to_gguf/export.py
+++ b/auto_round/export/export_to_gguf/export.py
@@ -22,6 +22,7 @@ from pathlib import Path
import requests
import torch
+from auto_round import envs
from auto_round.export.export_to_gguf.config import ModelType
from auto_round.export.export_to_gguf.convert import is_mmproj_tensor_name, wrapper_model_instance
from auto_round.export.export_to_gguf.llama_cpp_conversion import get_conversion
@@ -92,23 +93,6 @@ def _set_mmproj_output_path(model_instance):
return model_instance
-def _create_conversion_model(model_class, hparams, **kwargs):
- if not getattr(model_class, "supports_mtp_export", False):
- return model_class(hparams=hparams, **kwargs)
-
- # AutoRound exports the target model only; unlike llama.cpp's CLI, it does not
- # provide a separate MTP export mode. Conversion filters read this flag from
- # the concrete class, so restore it immediately after constructing the instance.
- original_no_mtp = model_class.no_mtp
- try:
- model_class.no_mtp = True
- model_instance = model_class(hparams=hparams, **kwargs)
- model_instance.no_mtp = True
- return model_instance
- finally:
- model_class.no_mtp = original_no_mtp
-
-
def create_model_class(
output_dir,
model,
@@ -147,9 +131,13 @@ def create_model_class(
output_type = FTYPE_MAP.get(output_type.lower())
hparams.pop("quantization_config", None)
- model_instance = _create_conversion_model(
- model_class,
- hparams,
+ if envs.AR_DISABLE_GGUF_MTP_EXPORT and getattr(model_class, "supports_mtp_export", False):
+ model_class = type(
+ f"AutoRound{model_class.__name__}",
+ (model_class,),
+ {"model_arch": model_class.model_arch, "no_mtp": True},
+ )
+ model_instance = model_class(
dir_model=Path(tmp_work_dir),
ftype=output_type,
fname_out=Path(output_dir),
@@ -159,6 +147,7 @@ def create_model_class(
split_max_size=0,
dry_run=False,
small_first_shard=False,
+ hparams=hparams,
)
if native_nontext_export:
logger.info("Using native llama.cpp F32 export for non-text GGUF model")
diff --git a/docs/environments.md b/docs/environments.md
index 6bf91f804..a9ceb17ae 100644
--- a/docs/environments.md
+++ b/docs/environments.md
@@ -79,6 +79,16 @@ export AR_DISABLE_OFFLOAD=1
export AR_DISABLE_DATASET_SUBPROCESS=true
```
+### AR_DISABLE_GGUF_MTP_EXPORT
+- **Description**: Disables multi-token prediction (MTP) layers when exporting GGUF models. By default, AutoRound follows the llama.cpp converter and includes MTP layers when the model provides them.
+- **Default**: `False` (equivalent to `"0"`)
+- **Valid Values**: `"1"`, `"true"`, `"yes"`, or `"on"` (case-insensitive) disable MTP export; any other value keeps MTP export enabled
+- **Usage**: Enable this when exporting a checkpoint that uses an MTP-capable architecture but does not contain MTP tensors
+
+```bash
+export AR_DISABLE_GGUF_MTP_EXPORT=1
+```
+
### AR_ACT_SCALE
- **Description**: Only for research. Controls the scaling factor applied to activation min/max values during activation quantization. A value less than 1.0 shrinks the clipping range, which can reduce outlier impact.
- **Default**: `1.0`
diff --git a/docs/environments_CN.md b/docs/environments_CN.md
index 7779bede7..27ac26672 100644
--- a/docs/environments_CN.md
+++ b/docs/environments_CN.md
@@ -79,6 +79,16 @@ export AR_DISABLE_OFFLOAD=1
export AR_DISABLE_DATASET_SUBPROCESS=true
```
+### AR_DISABLE_GGUF_MTP_EXPORT
+- **描述**:导出 GGUF 模型时禁用多 token 预测(MTP)层。默认情况下,AutoRound 遵循 llama.cpp 转换器的行为,在模型提供 MTP 层时将其一并导出。
+- **默认值**:`False`(等价于 `"0"`)
+- **有效值**:`"1"`、`"true"`、`"yes"` 或 `"on"`(不区分大小写)表示禁用 MTP 导出;其他值保持启用 MTP 导出
+- **用途**:当 checkpoint 使用支持 MTP 的模型架构,但实际不包含 MTP tensor 时启用
+
+```bash
+export AR_DISABLE_GGUF_MTP_EXPORT=1
+```
+
### AR_ACT_SCALE
- **描述**:只用于研究性质,控制激活量化时对激活值最小/最大值的缩放系数。小于 1.0 的值会缩小裁剪范围,有助于减小离群值的影响。
- **默认值**:`1.0`
diff --git a/test/unit/common/export/test_gguf_conversion.py b/test/unit/common/export/test_gguf_conversion.py
index b7ee0f8bc..aa3de4016 100644
--- a/test/unit/common/export/test_gguf_conversion.py
+++ b/test/unit/common/export/test_gguf_conversion.py
@@ -113,6 +113,68 @@ def _make_mock_model(cls, hparams=None):
return obj
[email protected]("disable_mtp_export", [False, True])
+def test_qwen35_gguf_mtp_export_control(monkeypatch, tmp_path, disable_mtp_export):
+ from auto_round.export.export_to_gguf import export
+ from auto_round.export.export_to_gguf.config import ModelType
+ from auto_round.export.export_to_gguf.conversion.base import ModelBase as ConversionModelBase
+ from auto_round.export.export_to_gguf.conversion.qwen import Qwen3_5TextModel
+
+ hparams = {
+ "architectures": ["Qwen3_5ForConditionalGeneration"],
+ "num_hidden_layers": 32,
+ "mtp_num_hidden_layers": 0 if disable_mtp_export else 1,
+ }
+
+ class FakeModel:
+ name_or_path = str(tmp_path)
+ config = type("Config", (), {"model_type": "qwen3_5"})()
+
+ class FakeConversion:
+ ModelBase = ConversionModelBase
+
+ @staticmethod
+ def model_type(model_type):
+ return model_type
+
+ @staticmethod
+ def get_model_architecture(_hparams, _model_type):
+ return "Qwen3_5ForConditionalGeneration"
+
+ @staticmethod
+ def get_model_class(_model_architecture, model_type):
+ return Qwen3_5TextModel
+
+ monkeypatch.setattr(ConversionModelBase, "load_hparams", staticmethod(lambda *_args: hparams.copy()))
+ monkeypatch.setattr(export, "get_conversion", lambda *_args, **_kwargs: FakeConversion)
+ monkeypatch.setattr(export, "wrapper_model_instance", lambda model_instance, **_kwargs: model_instance)
+ monkeypatch.setattr(export, "handle_special_model", lambda model_instance, _architecture: model_instance)
+ if disable_mtp_export:
+ monkeypatch.setenv("AR_DISABLE_GGUF_MTP_EXPORT", "1")
+ else:
+ monkeypatch.delenv("AR_DISABLE_GGUF_MTP_EXPORT", raising=False)
+
+ model = export.create_model_class(tmp_path, FakeModel(), {}, model_type=ModelType.TEXT)
+ item = ("mtp.layers.0.mlp.down_proj.weight", lambda: torch.ones(1, 1))
+
+ filtered = model.filter_tensors(item)
+ if disable_mtp_export:
+ assert model.block_count == 32
+ assert filtered is None
+ assert Qwen3_5TextModel.no_mtp is False
+ return
+
+ assert filtered is not None
+ name, tensor_fn = filtered
+
+ outputs = list(model.modify_tensors(tensor_fn(), name, bid=32))
+
+ assert model.block_count == 33
+ assert len(outputs) == 1
+ assert outputs[0][0] == "blk.32.ffn_down.weight"
+ assert torch.equal(outputs[0][1], tensor_fn())
+
+
# ==============================================================================
# granite.py tests
# ==============================================================================
diff --git a/test/unit/test_cpu/export/test_gguf_format.py b/test/unit/test_cpu/export/test_gguf_format.py
index 357f08661..ee2379d3f 100644
--- a/test/unit/test_cpu/export/test_gguf_format.py
+++ b/test/unit/test_cpu/export/test_gguf_format.py
@@ -422,48 +422,6 @@ class TestGGUF:
assert model_instance.ftype is ftypes["q4_0"]
assert model_instance.fname_out == tmp_path
- def test_autoround_export_disables_mtp_for_supported_conversion(self, tmp_path):
- from auto_round.export.export_to_gguf import export
-
- class FakeMtpModel:
- supports_mtp_export = True
- no_mtp = False
-
- def __init__(self, hparams, **kwargs):
- assert type(self).no_mtp
- self.hparams = hparams
- self.__dict__.update(kwargs)
-
- instance = export._create_conversion_model(
- FakeMtpModel,
- {"num_hidden_layers": 4},
- dir_model=tmp_path,
- )
-
- assert instance.no_mtp
- assert isinstance(instance, FakeMtpModel)
- assert FakeMtpModel.no_mtp is False
-
- def test_autoround_export_keeps_non_mtp_conversion_unchanged(self, tmp_path):
- from auto_round.export.export_to_gguf import export
-
- class FakeModel:
- supports_mtp_export = False
- no_mtp = False
-
- def __init__(self, hparams, **kwargs):
- assert not type(self).no_mtp
- self.hparams = hparams
- self.__dict__.update(kwargs)
-
- instance = export._create_conversion_model(
- FakeModel,
- {"num_hidden_layers": 4},
- dir_model=tmp_path,
- )
-
- assert instance.no_mtp is False
-
def test_qtype_setting(self, tiny_qwen_vl_model_path):
# Qwen2.5-0.5B-Instruct no output, token_embed q6_k fallbakc to q8_0 336M
# Qwen3-0.6B output q6_k, token_embed q4_0 448M
diff --git a/test/unit/test_cuda/export/test_gguf_format.py b/test/unit/test_cuda/export/test_gguf_format.py
index 5c929db14..ffa8d7562 100644
--- a/test/unit/test_cuda/export/test_gguf_format.py
+++ b/test/unit/test_cuda/export/test_gguf_format.py
@@ -103,7 +103,8 @@ class TestAutoRound:
@pytest.mark.skipif(not torch.cuda.is_available(), reason="requires CUDA")
@require_gguf
@pytest.mark.timeout(240)
- def test_qwen35_moe_gguf(self, tiny_qwen35_moe_model_path):
+ def test_qwen35_moe_gguf(self, tiny_qwen35_moe_model_path, monkeypatch):
+ monkeypatch.setenv("AR_DISABLE_GGUF_MTP_EXPORT", "1")
self._export_qwen35_moe_gguf(tiny_qwen35_moe_model_path)
@pytest.mark.skip(