PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度
fix cuda ut fail for transformers v5.0 (#1357)
Heng Guoeb2ccdc3df2c
diff --git a/auto_round/eval/eval_cli.py b/auto_round/eval/eval_cli.pyindex 5d6d965b0..938be51c6 100644--- a/auto_round/eval/eval_cli.py+++ b/auto_round/eval/eval_cli.py@@ -422,6 +422,8 @@ def _evaluate_tasks_with_retry(tasks, hflm, device_str, batch_size, limit, retrycuda_error_msg = traceback.format_exc()try:ori_batch_sizes = hflm.batch_sizes if hflm.batch_sizes else {"0": 64}+ if not hflm.batch_sizes:+ hflm.batch_sizes = ori_batch_sizes.copy()try:for k, v in hflm.batch_sizes.items():hflm.batch_sizes[k] = max(v // 2, 1)diff --git a/auto_round/inference/convert_model.py b/auto_round/inference/convert_model.pyindex d83f8bc5c..096836df1 100644--- a/auto_round/inference/convert_model.py+++ b/auto_round/inference/convert_model.py@@ -504,8 +504,29 @@ def post_init(model: torch.nn.Module, used_backends: list[str]) -> None:# GPTQModel post-initif need_gptqmodel_init:+ from gptqmodel.quantization import METHOD # pylint: disable=E0401+ from gptqmodel.utils.importer import hf_select_quant_linear_v2 # pylint: disable=E0401+ from gptqmodel.utils.model import hf_convert_gptq_v1_to_v2_format # pylint: disable=E0401from gptqmodel.utils.model import hf_gptqmodel_post_init as gptq_post_init # pylint: disable=E0401+ quant_linear = hf_select_quant_linear_v2(+ bits=model.config.quantization_config.bits,+ group_size=model.config.quantization_config.group_size,+ desc_act=False,+ sym=model.config.quantization_config.sym,+ format="gptq_v2",+ quant_method=METHOD.GPTQ,+ backend="EXLLAMA_V2",+ pack=True,+ device_map="auto",+ )+ model, _ = hf_convert_gptq_v1_to_v2_format(+ model,+ bits=model.config.quantization_config.bits,+ qlinear_kernel=quant_linear,+ checkpoint_format="gptq",+ meta=None,+ )model = gptq_post_init(model, use_act_order=False)# IPEX post-initdiff --git a/auto_round/utils/common.py b/auto_round/utils/common.pyindex 045f76936..b8d7ab9fb 100644--- a/auto_round/utils/common.py+++ b/auto_round/utils/common.py@@ -106,6 +106,11 @@ def monkey_patch_transformers():exc,)return++ if parsed_version >= version.parse("5.0.0"):+ from transformers.initialization import no_init_weights++ setattr(transformers.modeling_utils, "no_init_weights", no_init_weights)if parsed_version >= version.parse("4.56.0"):transformers.AutoModelForCausalLM.from_pretrained = rename_kwargs(torch_dtype="dtype")(transformers.AutoModelForCausalLM.from_pretraineddiff --git a/test/test_ark/requirements.txt b/test/test_ark/requirements.txtindex 286955dfd..fe43c33f0 100644--- a/test/test_ark/requirements.txt+++ b/test/test_ark/requirements.txt@@ -1,2 +1,2 @@auto-round-kernel-lm_eval @ git+https://github.com/EleutherAI/lm-evaluation-harness.git@main\ No newline at end of file+lm_eval >= 0.4.10 # for transformers >= 5.0.0\ No newline at end of filediff --git a/test/test_cpu/models/test_moe_model.py b/test/test_cpu/models/test_moe_model.pyindex 885436686..1125ab56d 100644--- a/test/test_cpu/models/test_moe_model.py+++ b/test/test_cpu/models/test_moe_model.py@@ -35,6 +35,10 @@ def setup_llama4():model_name = llama4_name_or_pathtokenizer = AutoTokenizer.from_pretrained(model_name, trust_remote_code=True)config = AutoConfig.from_pretrained(model_name, trust_remote_code=True)++ # TODO: Remove after https://github.com/huggingface/transformers/issues/43525 is resolved+ config.pad_token_id = None+config.vision_config.num_hidden_layers = 1 # Reduce layers for testingconfig.text_config.num_hidden_layers = 1# config.vision_config.rope_theta = config.vision_config.rope_parameters["rope_theta"] # for transformers >= 5.0@@ -123,10 +127,6 @@ def test_gptoss(setup_gpt_oss, scheme):shutil.rmtree(output_dir, ignore_errors=True)- transformers_version >= version.parse("5.0.0"),- reason="transformers v5 'Llama4VisionConfig' object has no attribute 'rope_theta'",-)def test_llama4(setup_llama4):model, tokenizer, output_dir, config = setup_llama4diff --git a/test/test_cpu/requirements.txt b/test/test_cpu/requirements.txtindex dbe2d5014..a6ce699a7 100644--- a/test/test_cpu/requirements.txt+++ b/test/test_cpu/requirements.txt@@ -6,6 +6,6 @@ torchvisionpillownumballmcompressor @ git+https://github.com/vllm-project/llm-compressor.git@main-lm_eval @ git+https://github.com/EleutherAI/lm-evaluation-harness.git@main+lm_eval >= 0.4.10 # for transformers >= 5.0.0diffusersprotobufdiff --git a/test/test_cuda/advanced/test_fp8_input.py b/test/test_cuda/advanced/test_fp8_input.pyindex ec3dc6bf3..de4dfa453 100644--- a/test/test_cuda/advanced/test_fp8_input.py+++ b/test/test_cuda/advanced/test_fp8_input.py@@ -4,13 +4,14 @@ import shutilimport pytestimport torchimport transformers+from packaging import versionfrom transformers import AutoModelForCausalLM, AutoTokenizerfrom auto_round import AutoRoundfrom auto_round.eval.evaluation import simple_evaluatefrom auto_round.utils import llm_load_model-from ...helpers import get_model_path, get_tiny_model+from ...helpers import get_model_path, get_tiny_model, transformers_versionclass TestAutoRound:@@ -46,6 +47,11 @@ class TestAutoRound:print(tokenizer.decode(model.generate(**inputs, max_new_tokens=50)[0]))shutil.rmtree(self.save_dir, ignore_errors=True)+ @pytest.mark.skipif(+ transformers_version >= version.parse("5.0.0"),+ reason="GGUF format saving and loading failed in transformers v5, \+ https://github.com/huggingface/transformers/issues/43482",+ )def test_gguf_imatrix(self):model, tokenizer = self.tiny_fp8_model()ar = AutoRound(model=model, tokenizer=tokenizer, iters=0)@@ -108,6 +114,11 @@ class TestAutoRound:shutil.rmtree(self.save_dir, ignore_errors=True)+ @pytest.mark.skipif(+ transformers_version >= version.parse("5.0.0"),+ reason="GGUF format saving and loading failed in transformers v5, \+ https://github.com/huggingface/transformers/issues/43482",+ )def test_fp8_model_gguf(self):from llama_cpp import Llama@@ -133,6 +144,11 @@ class TestAutoRound:print(output)shutil.rmtree(self.save_dir, ignore_errors=True)+ @pytest.mark.skipif(+ transformers_version >= version.parse("5.0.0"),+ reason="We need this patch for fp8 model loading without dequantization."+ "https://github.com/intel/auto-round/blob/72e1cecb4a984db101e26700618266115029b9ac/test/test_cuda/quantization/test_mxfp_nvfp.py#L19C5-L19C25",+ )def test_diff_datatype(self):for scheme in ["NVFP4", "MXFP4"]:model_name = get_model_path("qwen/Qwen3-0.6B-FP8")diff --git a/test/test_cuda/export/test_export.py b/test/test_cuda/export/test_export.pyindex acbe19482..25f29a4c8 100644--- a/test/test_cuda/export/test_export.py+++ b/test/test_cuda/export/test_export.py@@ -4,12 +4,13 @@ import shutilimport pytestimport torchimport transformers+from packaging import versionfrom transformers import AutoConfig, AutoModelForCausalLM, AutoRoundConfig, AutoTokenizerfrom auto_round import AutoRoundfrom auto_round.testing_utils import require_awq, require_optimum, require_package_version_ut-from ...helpers import get_model_path, get_tiny_model+from ...helpers import get_model_path, get_tiny_model, transformers_versionclass TestAutoRound:@@ -294,6 +295,10 @@ class TestAutoRound:print(tokenizer.decode(model.generate(**inputs, max_new_tokens=50)[0]))shutil.rmtree("./saved", ignore_errors=True)+ @pytest.mark.skipif(+ transformers_version >= version.parse("5.0"),+ reason="PhiConfig missing pad_token_id, https://github.com/huggingface/transformers/pull/43453",+ )def test_awq_lmhead_export(self, dataloader):bits, sym, group_size = 4, False, 128model_name = get_model_path("microsoft/phi-2")@@ -321,7 +326,11 @@ class TestAutoRound:assert isinstance(lm_head, WQLinear_GEMM), "Illegal AWQ quantization for lm_head layer"shutil.rmtree(quantized_model_path, ignore_errors=True)- def test_gptq_lmhead_export(self, tiny_qwen_model_path, dataloader):+ @pytest.mark.skipif(+ transformers_version >= version.parse("5.0"),+ reason="PhiConfig missing pad_token_id, https://github.com/huggingface/transformers/pull/43453",+ )+ def test_gptq_lmhead_export(self, dataloader):bits, sym, group_size = 4, True, 128model_name = get_model_path("microsoft/phi-2")tiny_model = get_tiny_model(model_name)diff --git a/test/test_cuda/export/test_gguf.py b/test/test_cuda/export/test_gguf.pyindex 09e7451c3..e71af74c4 100644--- a/test/test_cuda/export/test_gguf.py+++ b/test/test_cuda/export/test_gguf.py@@ -5,17 +5,23 @@ import sysimport pytestimport torchimport transformers+from packaging import versionfrom transformers import AutoModelForCausalLM, AutoTokenizerfrom auto_round import AutoRoundfrom auto_round.testing_utils import require_gguf-from ...helpers import get_model_path, get_tiny_model, save_tiny_model+from ...helpers import get_model_path, get_tiny_model, save_tiny_model, transformers_versionAUTO_ROUND_PATH = __file__.split("/")AUTO_ROUND_PATH = "/".join(AUTO_ROUND_PATH[: AUTO_ROUND_PATH.index("test")])+ transformers_version >= version.parse("5.0.0"),+ reason="GGUF format saving and loading failed in transformers v5, \+ https://github.com/huggingface/transformers/issues/43482",+)class TestAutoRound:save_dir = "./saved"diff --git a/test/test_cuda/integrations/test_transformers.py b/test/test_cuda/integrations/test_transformers.pyindex 638cb7396..9544a20a3 100644--- a/test/test_cuda/integrations/test_transformers.py+++ b/test/test_cuda/integrations/test_transformers.py@@ -50,6 +50,10 @@ class TestAutoRound:EXPECTED_OUTPUTS.add("There is a girl who likes adventure, and she has been exploring the world for many years. She has visited every country in Europe and has even traveled to some of the most remote parts of Africa. She enjoys hiking through the mountains and discovering")+ # transformers 5.0.0+ EXPECTED_OUTPUTS.add(+ "There is a girl who likes adventure, and she has been exploring the world for many years. She has visited every country in Europe and has even traveled to some of the most remote parts of Africa. She has also climbed mountains, swum"+ )device_map = "cuda"@@ -98,7 +102,8 @@ class TestAutoRound:)output = quantized_model.generate(**input_ids, max_new_tokens=40, do_sample=False)- assert self.tokenizer.decode(output[0], skip_special_tokens=True) in self.EXPECTED_OUTPUTS+ output_sentence = self.tokenizer.decode(output[0], skip_special_tokens=True)+ assert output_sentence in self.EXPECTED_OUTPUTS@require_intel_extension_for_pytorchdef test_quantized_model_on_cpu(self):diff --git a/test/test_cuda/integrations/test_vllm.py b/test/test_cuda/integrations/test_vllm.pyindex 686640573..99ed5fc05 100644--- a/test/test_cuda/integrations/test_vllm.py+++ b/test/test_cuda/integrations/test_vllm.py@@ -7,10 +7,14 @@ Validating the configuration and printing results for manual checking.Run `pytest test/test_cuda/test_vllm.py`."""+import os+import pytestfrom vllm import LLM, SamplingParamsfrom vllm.platforms import current_platform+os.environ["VLLM_WORKER_MULTIPROC_METHOD"] = "spawn"+MODELS = ["OPEA/Qwen2.5-0.5B-Instruct-int4-sym-inc", ##auto_round:auto_gptq"Intel/Qwen2-0.5B-Instruct-int4-sym-AutoRound", ##auto_round:auto_awqdiff --git a/test/test_cuda/models/test_diffusion.py b/test/test_cuda/models/test_diffusion.pyindex a3a90d14e..80bc238e8 100644--- a/test/test_cuda/models/test_diffusion.py+++ b/test/test_cuda/models/test_diffusion.py@@ -5,12 +5,14 @@ import shutilimport pytestimport requests-from diffusers import AutoPipelineForText2Image+from packaging import versionfrom PIL import Imagefrom auto_round import AutoRoundDiffusionfrom auto_round.testing_utils import require_gptqmodel, require_optimum, require_vlm_env+from ...helpers import transformers_version+class TestAutoRound:model_name = "/dataset/FLUX.1-dev"@@ -29,7 +31,13 @@ class TestAutoRound:shutil.rmtree("runs", ignore_errors=True)@require_optimum+ @pytest.mark.skipif(+ transformers_version >= version.parse("5.0.0"),+ reason="cannot import name 'MT5Tokenizer' from 'transformers', https://github.com/huggingface/diffusers/issues/13035",+ )def test_diffusion_tune(self):+ from diffusers import AutoPipelineForText2Image+## load the modelpipe = AutoPipelineForText2Image.from_pretrained(self.model_name).to("cuda")model = pipe.transformer@@ -57,7 +65,13 @@ class TestAutoRound:# skip model saving since it takes much timeautoround.quantize()+ @pytest.mark.skipif(+ transformers_version >= version.parse("5.0.0"),+ reason="cannot import name 'MT5Tokenizer' from 'transformers', https://github.com/huggingface/diffusers/issues/13035",+ )def test_diffusion_rtn(self):+ from diffusers import AutoPipelineForText2Image+## load the modelpipe = AutoPipelineForText2Image.from_pretrained(self.model_name)diff --git a/test/test_cuda/models/test_get_block_name.py b/test/test_cuda/models/test_get_block_name.pyindex 829ac1e46..1850ec5c7 100644--- a/test/test_cuda/models/test_get_block_name.py+++ b/test/test_cuda/models/test_get_block_name.py@@ -4,7 +4,7 @@ import shutilimport pytestimport torchimport transformers-from diffusers import AutoPipelineForText2Image+from packaging import versionfrom transformers import (AutoModelForCausalLM,AutoModelForVision2Seq,@@ -17,6 +17,8 @@ from transformers import (from auto_round import AutoRoundfrom auto_round.utils import get_block_names, is_pure_text_model+from ...helpers import transformers_version+class TestAutoRound:@classmethod@@ -186,7 +188,13 @@ class TestAutoRound:)assert not is_pure_text_model(model)+ @pytest.mark.skipif(+ transformers_version >= version.parse("5.0.0"),+ reason="cannot import name 'MT5Tokenizer' from 'transformers', https://github.com/huggingface/diffusers/issues/13035",+ )def test_flux(self):+ from diffusers import AutoPipelineForText2Image+model_name = "/dataset/FLUX.1-dev"pipe = AutoPipelineForText2Image.from_pretrained(model_name)model = pipe.transformerdiff --git a/test/test_cuda/models/test_moe_model.py b/test/test_cuda/models/test_moe_model.pyindex f6fc5af4b..40c545015 100644--- a/test/test_cuda/models/test_moe_model.py+++ b/test/test_cuda/models/test_moe_model.py@@ -30,7 +30,10 @@ def setup_llama4():model_name = "/dataset/Llama-4-Scout-17B-16E-Instruct"tokenizer = AutoTokenizer.from_pretrained(model_name, trust_remote_code=True)config = AutoConfig.from_pretrained(model_name, trust_remote_code=True)- config.pad_token_id = None # TODO: https://github.com/huggingface/transformers/issues/43525++ # TODO: Remove after https://github.com/huggingface/transformers/issues/43525 is resolved+ config.pad_token_id = None+config.vision_config.num_hidden_layers = 1 # Reduce layers for testingconfig.text_config.num_hidden_layers = 1model = Llama4ForConditionalGeneration(config)@@ -44,6 +47,10 @@ def setup_qwen3_vl_moe():model_name = "/models/Qwen3-VL-30B-A3B-Instruct"tokenizer = AutoTokenizer.from_pretrained(model_name)config = AutoConfig.from_pretrained(model_name)++ # TODO: Remove after https://github.com/huggingface/transformers/pull/43453 is merged+ config.text_config.pad_token_id = None+config.vision_config.num_hidden_layers = 1config.text_config.num_hidden_layers = 1config.num_hidden_layers = 1 # Reduce layers for testingdiff --git a/test/test_cuda/models/test_support_vlms.py b/test/test_cuda/models/test_support_vlms.pyindex cbe0ca19b..798674c83 100644--- a/test/test_cuda/models/test_support_vlms.py+++ b/test/test_cuda/models/test_support_vlms.py@@ -177,6 +177,7 @@ class TestSupportVLMS:print(response)shutil.rmtree(quantized_model_path, ignore_errors=True)+ @require_package_version_ut("transformers", "<4.54.0")def test_glm(self):model_path = "/models/glm-4v-9b/"## test tunediff --git a/test/test_cuda/models/test_vlms.py b/test/test_cuda/models/test_vlms.pyindex 087102da0..b0aa4a327 100644--- a/test/test_cuda/models/test_vlms.py+++ b/test/test_cuda/models/test_vlms.py@@ -147,7 +147,7 @@ class TestAutoRound:continueassert is_mllm_model(model)- for model_name in ["/models/glm-4-9b-chat", "/models/Qwen2.5-1.5B-Instruct/"]:+ for model_name in ["/models/Qwen2.5-1.5B-Instruct/"]:assert not is_mllm_model(model_name)model, _ = llm_load_model(model_name)assert not is_mllm_model(model)diff --git a/test/test_cuda/quantization/test_mxfp_nvfp.py b/test/test_cuda/quantization/test_mxfp_nvfp.pyindex 7aa3746a0..e2ca5b359 100644--- a/test/test_cuda/quantization/test_mxfp_nvfp.py+++ b/test/test_cuda/quantization/test_mxfp_nvfp.py@@ -1,5 +1,6 @@import copyimport shutil+from unittest.mock import patchimport pytestimport torch@@ -9,7 +10,14 @@ from transformers import AutoConfig, AutoModelForCausalLM, AutoTokenizerfrom auto_round import AutoRoundfrom auto_round.testing_utils import require_awq, require_optimum-from ...helpers import get_model_path, get_tiny_model+from ...helpers import get_model_path, save_tiny_model+++# Mock torch.cuda.get_device_capability to always return (9, 0) like H100+def mock_cuda_capability():+ with patch("torch.cuda.get_device_capability", return_value=(9, 0)):+ yieldclass TestAutoRound:@@ -28,18 +36,22 @@ class TestAutoRound:shutil.rmtree("./saved", ignore_errors=True)shutil.rmtree("runs", ignore_errors=True)- def test_fp8input_mxfp4_llmcompressor_format(self, dataloader):+ def test_fp8input_mxfp4_llmcompressor_format(self, dataloader, mock_cuda_capability):+ # mock_cuda_capability is not used but needed to enable fp8 support in torchmodel_name = get_model_path("qwen/Qwen3-0.6B-FP8")+ tiny_model_path = "./tmp/tiny_qwen3_fp8"+ save_tiny_model(model_name, tiny_model_path)scheme = "mxfp4"ar = AutoRound(- model=model_name,- iters=2,+ model=tiny_model_path,+ iters=0,seqlen=2,scheme=scheme,dataset=dataloader,)+ print(ar.model)compressed_model, _ = ar.quantize_and_save(output_dir=self.save_dir, format="llm_compressor")- tmp_layer = compressed_model.model.layers[3].self_attn.q_proj+ tmp_layer = compressed_model.model.layers[1].self_attn.q_projassert (hasattr(tmp_layer, "weight_scale")and hasattr(tmp_layer, "weight_packed")diff --git a/test/test_cuda/requirements.txt b/test/test_cuda/requirements.txtindex 3d4e3e101..3b2f3efd6 100644--- a/test/test_cuda/requirements.txt+++ b/test/test_cuda/requirements.txt@@ -3,7 +3,7 @@auto-gptqeinops# pip install -v git+https://github.com/ModelCloud/GPTQModel.git --no-build-isolation-lm_eval @ git+https://github.com/EleutherAI/lm-evaluation-harness.git@main+lm_eval >= 0.4.10 # for transformers >= 5.0.0optimumpandaspillowdiff --git a/test/test_cuda/schemes/test_scheme.py b/test/test_cuda/schemes/test_scheme.pyindex 315c5984b..255fdc80e 100644--- a/test/test_cuda/schemes/test_scheme.py+++ b/test/test_cuda/schemes/test_scheme.py@@ -2,11 +2,12 @@ import osimport shutilimport pytest+from packaging import versionfrom auto_round import AutoRoundfrom auto_round.schemes import QuantizationScheme-from ...helpers import get_model_path, save_tiny_model+from ...helpers import get_model_path, save_tiny_model, transformers_versionclass TestAutoRound:@@ -104,6 +105,9 @@ class TestAutoRound:if n == "model.decoder.layers.4.self_attn.k_proj":assert m.group_size == 64+ @pytest.mark.skipif(+ transformers_version >= version.parse("5.0.0"), reason="transformers v5 MOE model has breaking changes"+ )def test_q2k_mixed(self):model_path = "/data0/MiroThinker-v1.5-30B"saved_tiny_model_path = save_tiny_model(