PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度
fix gguf format mtp bug (#2297)
Heng Guo947ce102c2f3
diff --git a/auto_round/envs.py b/auto_round/envs.pyindex bef41f43d..a30d016e3 100644--- a/auto_round/envs.py+++ b/auto_round/envs.py@@ -23,6 +23,7 @@ if TYPE_CHECKING:AR_MODEL_FREE_SHARD_PARALLELISM: Optional[int] = NoneAUTO_ROUND_CACHE: Optional[str] = NoneAUTO_ROUND_GGUF_AUTO_UPDATE: bool = False+ AR_DISABLE_GGUF_MTP_EXPORT: bool = FalseLLAMA_CPP_ROOT: Optional[str] = NoneAR_AUTO_SCHEME_NSAMPLES: Optional[int] = NoneAR_AUTO_SCHEME_BATCH_SIZE: Optional[int] = None@@ -90,6 +91,8 @@ environment_variables: dict[str, Callable[[], Any]] = {"AUTO_ROUND_CACHE": lambda: os.getenv("AUTO_ROUND_CACHE", None),"AUTO_ROUND_GGUF_AUTO_UPDATE": lambda: os.getenv("AUTO_ROUND_GGUF_AUTO_UPDATE", "0").lower()in ("1", "true", "yes", "on"),+ "AR_DISABLE_GGUF_MTP_EXPORT": lambda: os.getenv("AR_DISABLE_GGUF_MTP_EXPORT", "0").lower()+ in ("1", "true", "yes", "on"),"LLAMA_CPP_ROOT": lambda: os.getenv("LLAMA_CPP_ROOT", None),# Controls the default number of calibration samples used by AutoScheme scoring# when ``AutoScheme.nsamples`` is not explicitly set.diff --git a/auto_round/export/export_to_gguf/export.py b/auto_round/export/export_to_gguf/export.pyindex 476dd1f83..443e8bc0c 100644--- a/auto_round/export/export_to_gguf/export.py+++ b/auto_round/export/export_to_gguf/export.py@@ -22,6 +22,7 @@ from pathlib import Pathimport requestsimport torch+from auto_round import envsfrom auto_round.export.export_to_gguf.config import ModelTypefrom auto_round.export.export_to_gguf.convert import is_mmproj_tensor_name, wrapper_model_instancefrom auto_round.export.export_to_gguf.llama_cpp_conversion import get_conversion@@ -92,23 +93,6 @@ def _set_mmproj_output_path(model_instance):return model_instance-def _create_conversion_model(model_class, hparams, **kwargs):- if not getattr(model_class, "supports_mtp_export", False):- return model_class(hparams=hparams, **kwargs)-- # AutoRound exports the target model only; unlike llama.cpp's CLI, it does not- # provide a separate MTP export mode. Conversion filters read this flag from- # the concrete class, so restore it immediately after constructing the instance.- original_no_mtp = model_class.no_mtp- try:- model_class.no_mtp = True- model_instance = model_class(hparams=hparams, **kwargs)- model_instance.no_mtp = True- return model_instance- finally:- model_class.no_mtp = original_no_mtp--def create_model_class(output_dir,model,@@ -147,9 +131,13 @@ def create_model_class(output_type = FTYPE_MAP.get(output_type.lower())hparams.pop("quantization_config", None)- model_instance = _create_conversion_model(- model_class,- hparams,+ if envs.AR_DISABLE_GGUF_MTP_EXPORT and getattr(model_class, "supports_mtp_export", False):+ model_class = type(+ f"AutoRound{model_class.__name__}",+ (model_class,),+ {"model_arch": model_class.model_arch, "no_mtp": True},+ )+ model_instance = model_class(dir_model=Path(tmp_work_dir),ftype=output_type,fname_out=Path(output_dir),@@ -159,6 +147,7 @@ def create_model_class(split_max_size=0,dry_run=False,small_first_shard=False,+ hparams=hparams,)if native_nontext_export:logger.info("Using native llama.cpp F32 export for non-text GGUF model")diff --git a/docs/environments.md b/docs/environments.mdindex 6bf91f804..a9ceb17ae 100644--- a/docs/environments.md+++ b/docs/environments.md@@ -79,6 +79,16 @@ export AR_DISABLE_OFFLOAD=1export AR_DISABLE_DATASET_SUBPROCESS=true```+### AR_DISABLE_GGUF_MTP_EXPORT+- **Description**: Disables multi-token prediction (MTP) layers when exporting GGUF models. By default, AutoRound follows the llama.cpp converter and includes MTP layers when the model provides them.+- **Default**: `False` (equivalent to `"0"`)+- **Valid Values**: `"1"`, `"true"`, `"yes"`, or `"on"` (case-insensitive) disable MTP export; any other value keeps MTP export enabled+- **Usage**: Enable this when exporting a checkpoint that uses an MTP-capable architecture but does not contain MTP tensors++```bash+export AR_DISABLE_GGUF_MTP_EXPORT=1+```+### AR_ACT_SCALE- **Description**: Only for research. Controls the scaling factor applied to activation min/max values during activation quantization. A value less than 1.0 shrinks the clipping range, which can reduce outlier impact.- **Default**: `1.0`diff --git a/docs/environments_CN.md b/docs/environments_CN.mdindex 7779bede7..27ac26672 100644--- a/docs/environments_CN.md+++ b/docs/environments_CN.md@@ -79,6 +79,16 @@ export AR_DISABLE_OFFLOAD=1export AR_DISABLE_DATASET_SUBPROCESS=true```+### AR_DISABLE_GGUF_MTP_EXPORT+- **描述**:导出 GGUF 模型时禁用多 token 预测(MTP)层。默认情况下,AutoRound 遵循 llama.cpp 转换器的行为,在模型提供 MTP 层时将其一并导出。+- **默认值**:`False`(等价于 `"0"`)+- **有效值**:`"1"`、`"true"`、`"yes"` 或 `"on"`(不区分大小写)表示禁用 MTP 导出;其他值保持启用 MTP 导出+- **用途**:当 checkpoint 使用支持 MTP 的模型架构,但实际不包含 MTP tensor 时启用++```bash+export AR_DISABLE_GGUF_MTP_EXPORT=1+```+### AR_ACT_SCALE- **描述**:只用于研究性质,控制激活量化时对激活值最小/最大值的缩放系数。小于 1.0 的值会缩小裁剪范围,有助于减小离群值的影响。- **默认值**:`1.0`diff --git a/test/unit/common/export/test_gguf_conversion.py b/test/unit/common/export/test_gguf_conversion.pyindex b7ee0f8bc..aa3de4016 100644--- a/test/unit/common/export/test_gguf_conversion.py+++ b/test/unit/common/export/test_gguf_conversion.py@@ -113,6 +113,68 @@ def _make_mock_model(cls, hparams=None):return obj[email protected]("disable_mtp_export", [False, True])+def test_qwen35_gguf_mtp_export_control(monkeypatch, tmp_path, disable_mtp_export):+ from auto_round.export.export_to_gguf import export+ from auto_round.export.export_to_gguf.config import ModelType+ from auto_round.export.export_to_gguf.conversion.base import ModelBase as ConversionModelBase+ from auto_round.export.export_to_gguf.conversion.qwen import Qwen3_5TextModel++ hparams = {+ "architectures": ["Qwen3_5ForConditionalGeneration"],+ "num_hidden_layers": 32,+ "mtp_num_hidden_layers": 0 if disable_mtp_export else 1,+ }++ class FakeModel:+ name_or_path = str(tmp_path)+ config = type("Config", (), {"model_type": "qwen3_5"})()++ class FakeConversion:+ ModelBase = ConversionModelBase++ @staticmethod+ def model_type(model_type):+ return model_type++ @staticmethod+ def get_model_architecture(_hparams, _model_type):+ return "Qwen3_5ForConditionalGeneration"++ @staticmethod+ def get_model_class(_model_architecture, model_type):+ return Qwen3_5TextModel++ monkeypatch.setattr(ConversionModelBase, "load_hparams", staticmethod(lambda *_args: hparams.copy()))+ monkeypatch.setattr(export, "get_conversion", lambda *_args, **_kwargs: FakeConversion)+ monkeypatch.setattr(export, "wrapper_model_instance", lambda model_instance, **_kwargs: model_instance)+ monkeypatch.setattr(export, "handle_special_model", lambda model_instance, _architecture: model_instance)+ if disable_mtp_export:+ monkeypatch.setenv("AR_DISABLE_GGUF_MTP_EXPORT", "1")+ else:+ monkeypatch.delenv("AR_DISABLE_GGUF_MTP_EXPORT", raising=False)++ model = export.create_model_class(tmp_path, FakeModel(), {}, model_type=ModelType.TEXT)+ item = ("mtp.layers.0.mlp.down_proj.weight", lambda: torch.ones(1, 1))++ filtered = model.filter_tensors(item)+ if disable_mtp_export:+ assert model.block_count == 32+ assert filtered is None+ assert Qwen3_5TextModel.no_mtp is False+ return++ assert filtered is not None+ name, tensor_fn = filtered++ outputs = list(model.modify_tensors(tensor_fn(), name, bid=32))++ assert model.block_count == 33+ assert len(outputs) == 1+ assert outputs[0][0] == "blk.32.ffn_down.weight"+ assert torch.equal(outputs[0][1], tensor_fn())++# ==============================================================================# granite.py tests# ==============================================================================diff --git a/test/unit/test_cpu/export/test_gguf_format.py b/test/unit/test_cpu/export/test_gguf_format.pyindex 357f08661..ee2379d3f 100644--- a/test/unit/test_cpu/export/test_gguf_format.py+++ b/test/unit/test_cpu/export/test_gguf_format.py@@ -422,48 +422,6 @@ class TestGGUF:assert model_instance.ftype is ftypes["q4_0"]assert model_instance.fname_out == tmp_path- def test_autoround_export_disables_mtp_for_supported_conversion(self, tmp_path):- from auto_round.export.export_to_gguf import export-- class FakeMtpModel:- supports_mtp_export = True- no_mtp = False-- def __init__(self, hparams, **kwargs):- assert type(self).no_mtp- self.hparams = hparams- self.__dict__.update(kwargs)-- instance = export._create_conversion_model(- FakeMtpModel,- {"num_hidden_layers": 4},- dir_model=tmp_path,- )-- assert instance.no_mtp- assert isinstance(instance, FakeMtpModel)- assert FakeMtpModel.no_mtp is False-- def test_autoround_export_keeps_non_mtp_conversion_unchanged(self, tmp_path):- from auto_round.export.export_to_gguf import export-- class FakeModel:- supports_mtp_export = False- no_mtp = False-- def __init__(self, hparams, **kwargs):- assert not type(self).no_mtp- self.hparams = hparams- self.__dict__.update(kwargs)-- instance = export._create_conversion_model(- FakeModel,- {"num_hidden_layers": 4},- dir_model=tmp_path,- )-- assert instance.no_mtp is False-def test_qtype_setting(self, tiny_qwen_vl_model_path):# Qwen2.5-0.5B-Instruct no output, token_embed q6_k fallbakc to q8_0 336M# Qwen3-0.6B output q6_k, token_embed q4_0 448Mdiff --git a/test/unit/test_cuda/export/test_gguf_format.py b/test/unit/test_cuda/export/test_gguf_format.pyindex 5c929db14..ffa8d7562 100644--- a/test/unit/test_cuda/export/test_gguf_format.py+++ b/test/unit/test_cuda/export/test_gguf_format.py@@ -103,7 +103,8 @@ class TestAutoRound:@pytest.mark.skipif(not torch.cuda.is_available(), reason="requires CUDA")@require_gguf@pytest.mark.timeout(240)- def test_qwen35_moe_gguf(self, tiny_qwen35_moe_model_path):+ def test_qwen35_moe_gguf(self, tiny_qwen35_moe_model_path, monkeypatch):+ monkeypatch.setenv("AR_DISABLE_GGUF_MTP_EXPORT", "1")self._export_qwen35_moe_gguf(tiny_qwen35_moe_model_path)@pytest.mark.skip(