PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度
Fix cpu ut for transformers v5 (#1333)
Tang Kaihuif6e7e3030d55
diff --git a/auto_round/eval/eval_cli.py b/auto_round/eval/eval_cli.pyindex 863a9110c..5d6d965b0 100644--- a/auto_round/eval/eval_cli.py+++ b/auto_round/eval/eval_cli.py@@ -279,11 +279,7 @@ def eval_task_by_task(device_str, parallelism = get_device_and_parallelism(device)# load after _eval_int in order to make sure import torch after set CUDA_VISIBLE_DEVICES- import traceback-- from lm_eval import simple_evaluate as lm_simple_evaluate # pylint: disable=E0401from lm_eval.models.huggingface import HFLM # pylint: disable=E0401- from transformers import AutoModelForCausalLM, AutoTokenizerfrom auto_round.utils import logger@@ -402,7 +398,7 @@ def _evaluate_tasks_with_retry(tasks, hflm, device_str, batch_size, limit, retryimport timeimport traceback- from lm_eval import simple_evaluate as lm_simple_evaluate # pylint: disable=E0401+ import lm_eval # pylint: disable=E0401from lm_eval.utils import make_table # pylint: disable=E0401from auto_round.utils import logger@@ -418,7 +414,7 @@ def _evaluate_tasks_with_retry(tasks, hflm, device_str, batch_size, limit, retrycurrent_retry_times = retry_timeswhile current_retry_times:try:- res = lm_simple_evaluate(+ res = lm_eval.simple_evaluate(model=hflm, model_args=None, device=device_str, tasks=task, batch_size=batch_size, limit=limit)break@@ -430,7 +426,7 @@ def _evaluate_tasks_with_retry(tasks, hflm, device_str, batch_size, limit, retryfor k, v in hflm.batch_sizes.items():hflm.batch_sizes[k] = max(v // 2, 1)logger.warning(f"Out of memory, reset batch_size to {hflm.batch_sizes} and re-try.")- res = lm_simple_evaluate(+ res = lm_eval.simple_evaluate(model=hflm, model_args=None, device=device_str, tasks=task, batch_size=1, limit=limit)hflm.batch_sizes = ori_batch_sizesdiff --git a/auto_round/eval/evaluation.py b/auto_round/eval/evaluation.pyindex 1f4df7b69..cff49b371 100644--- a/auto_round/eval/evaluation.py+++ b/auto_round/eval/evaluation.py@@ -31,7 +31,7 @@ def simple_evaluate_user_model(mllm: bool = False,**kwargs,):- from lm_eval import simple_evaluate as lm_simple_evaluate # pylint: disable=E0401+ import lm_eval # pylint: disable=E0401from lm_eval.models.huggingface import HFLM # pylint: disable=E0401if mllm:@@ -57,7 +57,7 @@ def simple_evaluate_user_model(dtype=eval_model_dtype,add_bos_token=add_bos_token,)- return lm_simple_evaluate(+ return lm_eval.simple_evaluate(model=hflm, model_args=None, batch_size=batch_size, max_batch_size=max_batch_size, limit=limit, **kwargs)@@ -71,9 +71,9 @@ def simple_evaluate(device: Optional[str] = None,**kwargs,):- from lm_eval import simple_evaluate as lm_simple_evaluate # pylint: disable=E0401+ import lm_eval # pylint: disable=E0401- return lm_simple_evaluate(+ return lm_eval.simple_evaluate(model=model,model_args=model_args,batch_size=batch_size,diff --git a/auto_round/modelling/gpt_oss.py b/auto_round/modelling/gpt_oss.pyindex 83d980d42..a71b5b610 100644--- a/auto_round/modelling/gpt_oss.py+++ b/auto_round/modelling/gpt_oss.py@@ -14,8 +14,16 @@import torch+import transformers+from packaging import versionfrom torch import nn-from transformers.modeling_utils import no_init_weights as skip_weights_initialize++transformers_version = version.parse(transformers.__version__)+if transformers_version < version.parse("5.0.0"):+ from transformers.modeling_utils import no_init_weights+else:+ from transformers.initialization import no_init_weights+from transformers.models.gpt_oss.configuration_gpt_oss import GptOssConfigfrom transformers.models.gpt_oss.modeling_gpt_oss import GptOssMLP@@ -71,7 +79,7 @@ class SequentialGPTOSSMoE(ReplacementModuleBase):# Build per-expert MLPsself.experts = nn.ModuleList()target_device = next(original.experts.parameters()).device- with skip_weights_initialize(), torch.device("meta"):+ with no_init_weights(), torch.device("meta"):for _ in range(E):self.experts.append(GPTOssSingleExpert(hidden_size, intermediate_size, dtype=dtype))diff --git a/auto_round/modelling/llama4.py b/auto_round/modelling/llama4.pyindex 795431554..7057f777d 100644--- a/auto_round/modelling/llama4.py+++ b/auto_round/modelling/llama4.py@@ -13,7 +13,14 @@# limitations under the License.# Note: adapted from # https://github.com/vllm-project/llm-compressor/blob/main/src/llmcompressor/modeling/llama4.pyimport torch-from transformers.modeling_utils import no_init_weights+import transformers+from packaging import version++transformers_version = version.parse(transformers.__version__)+if transformers_version < version.parse("5.0.0"):+ from transformers.modeling_utils import no_init_weights+else:+ from transformers.initialization import no_init_weightsfrom transformers.models.llama4.modeling_llama4 import Llama4Config, Llama4TextMLP, Llama4TextMoefrom auto_round.modelling.replace_modules import ReplacementModuleBasediff --git a/auto_round_extension/torch/qlinear_torch.py b/auto_round_extension/torch/qlinear_torch.pyindex e223f2aaa..d5389cfa6 100644--- a/auto_round_extension/torch/qlinear_torch.py+++ b/auto_round_extension/torch/qlinear_torch.py@@ -25,6 +25,25 @@ from auto_round.utils import get_packing_devicelogger = getLogger(__name__)+_DEVICE_WF_3BITS_TENSORS = {}+# Constants for FP4 values (E2M1 format)+_wf_3bits = [+ [0, 3, 6, 9, 12, 15, 18, 21, 24, 27, 30, 0],+ [0, 1, 4, 7, 10, 13, 16, 19, 22, 25, 28, 31],+ [0, 2, 5, 8, 11, 14, 17, 20, 23, 26, 29, 0],+]+++def get_wf_3bits_tensor(device):+ """Get device-specific wf_3bits tensor, creating it if needed."""+ device_str = str(device)+ if device_str not in _DEVICE_WF_3BITS_TENSORS:+ _DEVICE_WF_3BITS_TENSORS[device_str] = torch.tensor(_wf_3bits, dtype=torch.int32, device=device).reshape(+ 1, 3, 12+ )+ return _DEVICE_WF_3BITS_TENSORS[device_str]++class QuantLinear(nn.Module):"""Torch quantized linear layer.@@ -72,17 +91,11 @@ class QuantLinear(nn.Module):# is performed by unpacking the weights and using torch.matmulif self.bits in [2, 4, 8]:- self.wf = torch.tensor(list(range(0, 32, self.bits)), dtype=torch.int32).unsqueeze(0)- elif self.bits == 3:self.wf = torch.tensor(- [- [0, 3, 6, 9, 12, 15, 18, 21, 24, 27, 30, 0],- [0, 1, 4, 7, 10, 13, 16, 19, 22, 25, 28, 31],- [0, 2, 5, 8, 11, 14, 17, 20, 23, 26, 29, 0],- ],- dtype=torch.int32,- ).reshape(1, 3, 12)-+ list(range(0, 32, self.bits)), dtype=torch.int32, device=self.qweight.device+ ).unsqueeze(0)+ elif self.bits == 3:+ self.wf = get_wf_3bits_tensor(device=self.qweight.device)self.dequant_dtype = torch.int16 if self.bits == 8 else torch.int8def post_init(self):@@ -277,7 +290,9 @@ class QuantLinear(nn.Module):if self.bits in [2, 4, 8]:if self.wf.device != self.qzeros.device:- self.wf = self.wf.to(self.qzeros.device)+ self.wf = torch.tensor(+ list(range(0, 32, self.bits)), dtype=torch.int32, device=self.qzeros.device+ ).unsqueeze(0)zeros = torch.bitwise_right_shift(torch.unsqueeze(self.qzeros, 2).expand(-1, -1, 32 // self.bits),self.wf.unsqueeze(0),@@ -293,7 +308,7 @@ class QuantLinear(nn.Module):)elif self.bits == 3:if self.wf.device != self.qzeros.device:- self.wf = self.wf.to(self.qzeros.device)+ self.wf = get_wf_3bits_tensor(self.qzeros.device)zeros = self.qzeros.reshape(self.qzeros.shape[0], self.qzeros.shape[1] // 3, 3, 1).expand(-1, -1, -1, 12)zeros = zeros >> self.wf.unsqueeze(0)zeros[:, :, 0, 10] = (zeros[:, :, 0, 10] & 0x3) | ((zeros[:, :, 1, 0] << 2) & 0x4)@@ -335,7 +350,7 @@ class QuantLinear(nn.Module):out = torch.matmul(x, weights)out = out.to(x_dtype)out = out.reshape(out_shape)- out = out + self.bias if self.bias is not None else out+ out = (out + self.bias).to(x_dtype) if self.bias is not None else outreturn outdiff --git a/auto_round_extension/torch/qlinear_torch_zp.py b/auto_round_extension/torch/qlinear_torch_zp.pyindex f0701fc1f..e501066b3 100644--- a/auto_round_extension/torch/qlinear_torch_zp.py+++ b/auto_round_extension/torch/qlinear_torch_zp.py@@ -21,6 +21,7 @@ import torch.nn as nnimport transformersfrom auto_round.utils import get_packing_device+from auto_round_extension.torch.qlinear_torch import get_wf_3bits_tensorlogger = getLogger(__name__)@@ -72,16 +73,11 @@ class QuantLinear(nn.Module):# is performed by unpacking the weights and using torch.matmulif self.bits in [2, 4, 8]:- self.wf = torch.tensor(list(range(0, 32, self.bits)), dtype=torch.int32).unsqueeze(0)- else: ## bits == 3self.wf = torch.tensor(- [- [0, 3, 6, 9, 12, 15, 18, 21, 24, 27, 30, 0],- [0, 1, 4, 7, 10, 13, 16, 19, 22, 25, 28, 31],- [0, 2, 5, 8, 11, 14, 17, 20, 23, 26, 29, 0],- ],- dtype=torch.int32,- ).reshape(1, 3, 12)+ list(range(0, 32, self.bits)), dtype=torch.int32, device=self.qweight.device+ ).unsqueeze(0)+ else: ## bits == 3+ self.wf = get_wf_3bits_tensor(device=self.qweight.device)self.dequant_dtype = torch.int16 if self.bits == 8 else torch.int8@@ -276,7 +272,9 @@ class QuantLinear(nn.Module):if self.bits in [2, 4, 8]:if self.wf.device != self.qzeros.device:- self.wf = self.wf.to(self.qzeros.device)+ self.wf = torch.tensor(+ list(range(0, 32, self.bits)), dtype=torch.int32, device=self.qzeros.device+ ).unsqueeze(0)zeros = torch.bitwise_right_shift(torch.unsqueeze(self.qzeros, 2).expand(-1, -1, 32 // self.bits),self.wf.unsqueeze(0),@@ -292,7 +290,7 @@ class QuantLinear(nn.Module):)elif self.bits == 3:if self.wf.device != self.qzeros.device:- self.wf = self.wf.to(self.qzeros.device)+ self.wf = get_wf_3bits_tensor(device=self.qzeros.device)zeros = self.qzeros.reshape(self.qzeros.shape[0], self.qzeros.shape[1] // 3, 3, 1).expand(-1, -1, -1, 12)zeros = zeros >> self.wf.unsqueeze(0)zeros[:, :, 0, 10] = (zeros[:, :, 0, 10] & 0x3) | ((zeros[:, :, 1, 0] << 2) & 0x4)@@ -335,7 +333,7 @@ class QuantLinear(nn.Module):out = torch.matmul(x, weights)out = out.to(x_dtype)out = out.reshape(out_shape)- out = out + self.bias if self.bias is not None else out+ out = (out + self.bias).to(x_dtype) if self.bias is not None else outreturn outdiff --git a/test/fixtures.py b/test/fixtures.pyindex 624170710..cf9ad806e 100644--- a/test/fixtures.py+++ b/test/fixtures.py@@ -64,7 +64,7 @@ def tiny_phi2_model_path():def tiny_deepseek_v2_model_path():model_name_or_path = deepseek_v2_name_or_pathtiny_model_path = "./tmp/tiny_deepseek_v2_model_path"- tiny_model_path = save_tiny_model(model_name_or_path, tiny_model_path, num_layers=2)+ tiny_model_path = save_tiny_model(model_name_or_path, tiny_model_path, num_layers=2, trust_remote_code=False)yield tiny_model_pathshutil.rmtree(tiny_model_path, ignore_errors=True)diff --git a/test/helpers.py b/test/helpers.pyindex e8630e1fa..efd04dff5 100644--- a/test/helpers.py+++ b/test/helpers.py@@ -4,9 +4,12 @@ import osimport pytestimport torchimport transformers+from packaging import versionfrom auto_round.utils import get_attr, llm_load_model, mllm_load_model, set_attr+transformers_version = version.parse(transformers.__version__)+# Automatic choose local path or model name.def get_model_path(model_name: str) -> str:@@ -98,14 +101,14 @@ def save_tiny_model(model_name_or_path, tiny_model_path, num_layers=2, is_mllm=Ffor key in model._tied_weights_keys:weight = get_attr(model, key)set_attr(model, key, copy.deepcopy(weight))- tokenizer = transformers.AutoTokenizer.from_pretrained(model_name_or_path, trust_remote_code=True)+ tokenizer = transformers.AutoTokenizer.from_pretrained(model_name_or_path, **kwargs)test_path = os.path.dirname(__file__)tiny_model_path = os.path.join(test_path, tiny_model_path.removeprefix("./"))model.save_pretrained(tiny_model_path)tokenizer.save_pretrained(tiny_model_path)if is_mllm:- processor = transformers.AutoProcessor.from_pretrained(model_name_or_path, trust_remote_code=True)- image_processor = transformers.AutoImageProcessor.from_pretrained(model_name_or_path, trust_remote_code=True)+ processor = transformers.AutoProcessor.from_pretrained(model_name_or_path, **kwargs)+ image_processor = transformers.AutoImageProcessor.from_pretrained(model_name_or_path, **kwargs)processor.save_pretrained(tiny_model_path)image_processor.save_pretrained(tiny_model_path)print(f"[Fixture]: built tiny model path:{tiny_model_path} for testing in session")diff --git a/test/test_ark/requirements.txt b/test/test_ark/requirements.txtindex 07605745d..286955dfd 100644--- a/test/test_ark/requirements.txt+++ b/test/test_ark/requirements.txt@@ -1,2 +1,2 @@auto-round-kernel-lm-eval\ No newline at end of file+lm_eval @ git+https://github.com/EleutherAI/lm-evaluation-harness.git@main\ No newline at end of filediff --git a/test/test_cpu/core/test_autoround.py b/test/test_cpu/core/test_autoround.pyindex 79458e237..6e77a7b2f 100644--- a/test/test_cpu/core/test_autoround.py+++ b/test/test_cpu/core/test_autoround.py@@ -3,13 +3,14 @@ import shutilimport pytestimport torch+from packaging import versionfrom transformers import AutoModelForCausalLM, AutoRoundConfig, AutoTokenizerfrom auto_round import AutoRoundfrom auto_round.eval.evaluation import simple_evaluate_user_modelfrom auto_round.utils import get_module-from ...helpers import get_model_path, model_infer, opt_name_or_path, qwen_name_or_path+from ...helpers import get_model_path, model_infer, opt_name_or_path, qwen_name_or_path, transformers_versionclass TestAutoRound:@@ -72,6 +73,10 @@ class TestAutoRound:)autoround.quantize()+ @pytest.mark.skipif(+ transformers_version >= version.parse("5.0"),+ reason="PhiConfig missing pad_token_id, https://github.com/huggingface/transformers/pull/43453",+ )def test_consecutive_quant(self, tiny_opt_model_path, tiny_phi2_model_path, dataloader):bits, group_size, sym = 4, -1, Falseautoround = AutoRound(@@ -456,8 +461,12 @@ class TestAutoRound:model = Qwen2VLForConditionalGeneration.from_pretrained(model_name, quantization_config=quantization_config, device_map="cpu", torch_dtype=torch.float16)- assert isinstance(model.visual.blocks[0].attn.qkv, torch.nn.Linear)- assert not isinstance(model.visual.merger.mlp[0], QuantLinear)+ if transformers_version < version.parse("5.0.0"):+ assert isinstance(model.visual.blocks[0].attn.qkv, torch.nn.Linear)+ assert not isinstance(model.visual.merger.mlp[0], QuantLinear)+ else:+ assert isinstance(model.model.visual.blocks[0].attn.qkv, torch.nn.Linear)+ assert not isinstance(model.model.visual.merger.mlp[0], QuantLinear)if hasattr(model.model, "language_model"):assert isinstance(model.model.language_model.layers[0].self_attn.v_proj, QuantLinear)else:diff --git a/test/test_cpu/export/test_export.py b/test/test_cpu/export/test_export.pyindex cb6033de9..f7702108f 100644--- a/test/test_cpu/export/test_export.py+++ b/test/test_cpu/export/test_export.py@@ -3,11 +3,12 @@ import shutilimport pytestimport torch+from packaging import versionfrom transformers import AutoModelForCausalLM, AutoRoundConfig, AutoTokenizerfrom auto_round import AutoRound-from ...helpers import get_model_path, opt_name_or_path+from ...helpers import get_model_path, opt_name_or_path, transformers_versiondef _get_folder_size(path: str) -> float:@@ -262,7 +263,10 @@ class TestAutoRound:assert "model.decoder.layers.8.self_attn.v_scale" in f.keys()assert f.get_tensor("model.decoder.layers.5.self_attn.v_scale").shape == torch.Size([1])assert f.get_tensor("model.decoder.layers.5.self_attn.k_scale").shape == torch.Size([1])- assert f.get_tensor("model.decoder.layers.5.self_attn.k_scale").dtype == torch.float32+ assert (+ f.get_tensor("model.decoder.layers.5.self_attn.k_scale").dtype == torch.float32+ or f.get_tensor("model.decoder.layers.5.self_attn.k_scale").dtype == torch.bfloat16+ )shutil.rmtree(quantized_model_path, ignore_errors=True)model = AutoModelForCausalLM.from_pretrained(model_name, torch_dtype="auto", trust_remote_code=True)@@ -318,10 +322,14 @@ class TestAutoRound:weight_name = f"model.decoder.layers.8.self_attn.{attr}"assert weight_name in f.keys()assert f.get_tensor(weight_name).shape == torch.Size([1])- assert f.get_tensor(weight_name).dtype == torch.float32+ assert f.get_tensor(weight_name).dtype == torch.float32 or f.get_tensor(weight_name).dtype == torch.bfloat16shutil.rmtree(quantized_model_path, ignore_errors=True)+ @pytest.mark.skipif(+ transformers_version >= version.parse("5.0"),+ reason="PhiConfig missing pad_token_id, https://github.com/huggingface/transformers/pull/43453",+ )def test_awq_lmhead_export(self, dataloader):bits, sym, group_size = 4, False, 128model_name = get_model_path("microsoft/phi-2")@@ -348,6 +356,10 @@ class TestAutoRound:assert isinstance(lm_head, WQLinear_GEMM), "Illegal AWQ quantization for lm_head layer"shutil.rmtree(quantized_model_path, ignore_errors=True)+ @pytest.mark.skipif(+ transformers_version >= version.parse("5.0"),+ reason="PhiConfig missing pad_token_id, https://github.com/huggingface/transformers/pull/43453",+ )def test_gptq_lmhead_export(self, dataloader):bits, sym, group_size = 4, True, 128# Note that, to save UT tuning time, the local model is intentionally kept lightweight, using only 2 hidden layers.diff --git a/test/test_cpu/export/test_gguf_format.py b/test/test_cpu/export/test_gguf_format.pyindex 2e3cde6f0..f1980c455 100644--- a/test/test_cpu/export/test_gguf_format.py+++ b/test/test_cpu/export/test_gguf_format.py@@ -4,16 +4,22 @@ import sysimport pytestimport torch+from packaging import versionfrom transformers import AutoModelForCausalLM, AutoTokenizerfrom auto_round import AutoRound-from ...helpers import get_model_path, get_tiny_model, save_tiny_model+from ...helpers import get_model_path, get_tiny_model, save_tiny_model, transformers_versionAUTO_ROUND_PATH = __file__.split("/")AUTO_ROUND_PATH = "/".join(AUTO_ROUND_PATH[: AUTO_ROUND_PATH.index("test")])+ transformers_version >= version.parse("5.0.0"),+ reason="GGUF format saving and loading failed in transformers v5, \+ https://github.com/huggingface/transformers/issues/43482",+)class TestGGUF:@classmethod@@ -60,6 +66,12 @@ class TestGGUF:autoround.quantize_and_save(output_dir=quantized_model_path, inplace=False, format="gguf:q4_0")gguf_file = os.listdir(quantized_model_path)[0]++ # TODO: fix the issue of gguf loading error in transformers v5+ # cls = transformers.generation.configuration_utils.GenerationConfig'>, json_file = None+ # def _dict_from_json_file(cls, json_file: str | os.PathLike):+ # > with open(json_file, "r", encoding="utf-8") as reader:+ # E TypeError: expected str, bytes or os.PathLike object, not NoneTypemodel = AutoModelForCausalLM.from_pretrained(quantized_model_path, gguf_file=gguf_file, device_map="auto")text = "There is a girl who likes adventure,"inputs = self.tokenizer(text, return_tensors="pt").to(model.device)diff --git a/test/test_cpu/integrations/test_llmc_integration.py b/test/test_cpu/integrations/test_llmc_integration.pyindex cea412327..069d4fe06 100644--- a/test/test_cpu/integrations/test_llmc_integration.py+++ b/test/test_cpu/integrations/test_llmc_integration.py@@ -3,10 +3,13 @@ import torchfrom compressed_tensors.quantization import QuantizationArgs, QuantizationSchemefrom llmcompressor import oneshotfrom llmcompressor.modifiers.autoround import AutoRoundModifier+from packaging import versionfrom transformers import AutoModelForCausalLM, AutoTokenizerfrom auto_round.calib_dataset import get_dataset+from ...helpers import transformers_version+recipe_str = """quant_stage:quant_modifiers:@@ -39,6 +42,11 @@ recipe_modifier_full = AutoRoundModifier()+ transformers_version >= version.parse("5.0.0"),+ reason="transformers 5.0 use_auth_token is deprecated and llmcompressor oneshot has not been updated yet, \+ https://github.com/vllm-project/llm-compressor/issues/2289",+)@pytest.mark.parametrize("recipe",[diff --git a/test/test_cpu/models/test_diffusion.py b/test/test_cpu/models/test_diffusion.pyindex 32848ce85..ca495500a 100644--- a/test/test_cpu/models/test_diffusion.py+++ b/test/test_cpu/models/test_diffusion.py@@ -1,11 +1,11 @@import shutilimport pytest-from diffusers import AutoPipelineForText2Image+from packaging import versionfrom auto_round import AutoRound-from ...helpers import get_model_path+from ...helpers import get_model_path, transformers_versionflux_name_or_path = get_model_path("black-forest-labs/FLUX.1-dev")@@ -13,12 +13,18 @@ flux_name_or_path = get_model_path("black-forest-labs/FLUX.1-dev")@pytest.fixturedef setup_flux():"""Fixture to set up the Flux model and tokenizer."""+ from diffusers import AutoPipelineForText2Image+model_name = flux_name_or_pathpipe = AutoPipelineForText2Image.from_pretrained(model_name)output_dir = "./tmp/test_quantized_flux"return pipe, output_dir+ transformers_version >= version.parse("5.0.0"),+ reason="cannot import name 'MT5Tokenizer' from 'transformers', https://github.com/huggingface/diffusers/issues/13035",+)def test_flux(setup_flux):pipe, output_dir = setup_fluxautoround = AutoRound(diff --git a/test/test_cpu/models/test_moe_model.py b/test/test_cpu/models/test_moe_model.pyindex 4934d619e..8872edccb 100644--- a/test/test_cpu/models/test_moe_model.py+++ b/test/test_cpu/models/test_moe_model.py@@ -1,13 +1,14 @@import shutilimport pytest+from packaging import versionfrom transformers import AutoConfig, AutoProcessor, AutoTokenizer, Llama4ForConditionalGenerationfrom transformers.models.gpt_oss.modeling_gpt_oss import GptOssForCausalLMfrom transformers.models.qwen3_vl_moe.modeling_qwen3_vl_moe import Qwen3VLMoeForConditionalGenerationfrom auto_round import AutoRound-from ...helpers import get_model_path+from ...helpers import get_model_path, transformers_versiongpt_oss_name_or_path = get_model_path("unsloth/gpt-oss-20b-BF16")llama4_name_or_path = get_model_path("meta-llama/Llama-4-Scout-17B-16E-Instruct")@@ -36,6 +37,7 @@ def setup_llama4():config = AutoConfig.from_pretrained(model_name, trust_remote_code=True)config.vision_config.num_hidden_layers = 1 # Reduce layers for testingconfig.text_config.num_hidden_layers = 1+ # config.vision_config.rope_theta = config.vision_config.rope_parameters["rope_theta"] # for transformers >= 5.0model = Llama4ForConditionalGeneration(config)output_dir = "./tmp/test_quantized_llama4"return model, tokenizer, output_dir, config@@ -121,6 +123,10 @@ def test_gptoss(setup_gpt_oss, scheme):shutil.rmtree(output_dir, ignore_errors=True)+ transformers_version >= version.parse("5.0.0"),+ reason="transformers v5 'Llama4VisionConfig' object has no attribute 'rope_theta'",+)def test_llama4(setup_llama4):model, tokenizer, output_dir, config = setup_llama4diff --git a/test/test_cpu/quantization/test_mxfp_nvfp.py b/test/test_cpu/quantization/test_mxfp_nvfp.pyindex f5044bc73..3c2e9bcce 100644--- a/test/test_cpu/quantization/test_mxfp_nvfp.py+++ b/test/test_cpu/quantization/test_mxfp_nvfp.py@@ -3,11 +3,12 @@ import shutilimport pytestimport torch+from packaging import versionfrom transformers import AutoModelForCausalLM, AutoRoundConfig, AutoTokenizerfrom auto_round import AutoRound-from ...helpers import is_model_outputs_similar+from ...helpers import is_model_outputs_similar, transformers_versiondef _get_folder_size(path: str) -> float:@@ -31,6 +32,9 @@ class TestAutoRoundFP:shutil.rmtree("./saved", ignore_errors=True)shutil.rmtree("runs", ignore_errors=True)+ @pytest.mark.skipif(+ transformers_version >= version.parse("5.0.0"), reason="transformers v5 MOE model has breaking changes"+ )def test_nvfp4_moe_actmax_rtn(self, tiny_deepseek_v2_model_path, dataloader):model_name = tiny_deepseek_v2_model_pathlayer_config = {@@ -58,6 +62,9 @@ class TestAutoRoundFP:), "Illegal NVFP4 quantization for lm_head layer"shutil.rmtree(self.save_dir, ignore_errors=True)+ @pytest.mark.skipif(+ transformers_version >= version.parse("5.0.0"), reason="transformers v5 MOE model has breaking changes"+ )def test_nvfp4_moe_actmax_ar(self, tiny_deepseek_v2_model_path, dataloader):model_name = tiny_deepseek_v2_model_pathlayer_config = {@@ -90,6 +97,9 @@ class TestAutoRoundFP:assert is_model_outputs_similar(model_name, quantized_model_path)shutil.rmtree(self.save_dir, ignore_errors=True)+ @pytest.mark.skipif(+ transformers_version >= version.parse("5.0.0"), reason="transformers v5 MOE model has breaking changes"+ )def test_mxfp4_moe_ar(self, tiny_deepseek_v2_model_path, dataloader):model_name = tiny_deepseek_v2_model_pathlayer_config = {@@ -323,6 +333,9 @@ class TestAutoRoundFP:), "Illegal NVFP4 packing name or data_type or shape"shutil.rmtree(quantized_model_path, ignore_errors=True)+ @pytest.mark.skipif(+ transformers_version >= version.parse("5.0.0"), reason="transformers v5 MOE model has breaking changes"+ )def test_qwen_moe_quant_infer(self, tiny_qwen_moe_model_path, dataloader):model_name = tiny_qwen_moe_model_pathlayer_config = {diff --git a/test/test_cpu/requirements.txt b/test/test_cpu/requirements.txtindex 7f8f469f8..dbe2d5014 100644--- a/test/test_cpu/requirements.txt+++ b/test/test_cpu/requirements.txt@@ -6,6 +6,6 @@ torchvisionpillownumballmcompressor @ git+https://github.com/vllm-project/llm-compressor.git@main-lm_eval+lm_eval @ git+https://github.com/EleutherAI/lm-evaluation-harness.git@maindiffusersprotobufdiff --git a/test/test_cuda/models/test_moe_model.py b/test/test_cuda/models/test_moe_model.pyindex 66b0dc827..0cd2bd5e9 100644--- a/test/test_cuda/models/test_moe_model.py+++ b/test/test_cuda/models/test_moe_model.py@@ -31,6 +31,7 @@ def setup_llama4():model_name = "/dataset/Llama-4-Scout-17B-16E-Instruct"tokenizer = AutoTokenizer.from_pretrained(model_name, trust_remote_code=True)config = AutoConfig.from_pretrained(model_name, trust_remote_code=True)+ config.pad_token_id = None # TODO: https://github.com/huggingface/transformers/issues/43525config.vision_config.num_hidden_layers = 1 # Reduce layers for testingconfig.text_config.num_hidden_layers = 1model = Llama4ForConditionalGeneration(config)diff --git a/test/test_cuda/quantization/test_mix_bits.py b/test/test_cuda/quantization/test_mix_bits.pyindex 614a5d2b1..9daa9b727 100644--- a/test/test_cuda/quantization/test_mix_bits.py+++ b/test/test_cuda/quantization/test_mix_bits.py@@ -263,7 +263,13 @@ class TestAutoRound:sampling_params = SamplingParams(temperature=0.8, top_p=0.95)# Create an LLM.QUANTIZATION = "auto-round" # quantized_model_path- llm = LLM(model=quantized_model_path, quantization=QUANTIZATION, trust_remote_code=True, tensor_parallel_size=1)+ llm = LLM(+ model=quantized_model_path,+ quantization=QUANTIZATION,+ trust_remote_code=True,+ tensor_parallel_size=1,+ allow_deprecated_quantization=True,+ )outputs = llm.generate(prompts, sampling_params)# Print the outputs.for output in outputs:@@ -302,8 +308,12 @@ class TestAutoRound:# Create a sampling params object.sampling_params = SamplingParams(temperature=0.8, top_p=0.95)# Create an LLM.- QUANTIZATION = "auto-round" # quantized_model_path- llm = LLM(model=quantized_model_path, trust_remote_code=True, tensor_parallel_size=1)+ llm = LLM(+ model=quantized_model_path,+ trust_remote_code=True,+ tensor_parallel_size=1,+ allow_deprecated_quantization=True,+ )outputs = llm.generate(prompts, sampling_params)# Print the outputs.for output in outputs:diff --git a/test/test_cuda/quantization/test_mxfp_nvfp.py b/test/test_cuda/quantization/test_mxfp_nvfp.pyindex 922cb9f9b..7aa3746a0 100644--- a/test/test_cuda/quantization/test_mxfp_nvfp.py+++ b/test/test_cuda/quantization/test_mxfp_nvfp.py@@ -150,12 +150,12 @@ class TestAutoRound:layer_config=layer_config,)quantized_model_path = self.save_dir- autoround.quantize_and_save(output_dir=quantized_model_path, inplace=True, format="auto_round")+ autoround.quantize_and_save(output_dir=quantized_model_path, inplace=False, format="auto_round")model = AutoModelForCausalLM.from_pretrained(quantized_model_path, torch_dtype="auto", device_map="auto")tokenizer = AutoTokenizer.from_pretrained(quantized_model_path)from auto_round.eval.evaluation import simple_evaluate_user_model- result = simple_evaluate_user_model(model, tokenizer, batch_size=16, tasks="piqa")+ result = simple_evaluate_user_model(model, tokenizer, batch_size=16, tasks="piqa", limit=10)print(result["results"]["piqa"]["acc,none"])assert result["results"]["piqa"]["acc,none"] > 0.49shutil.rmtree(quantized_model_path, ignore_errors=True)diff --git a/test/test_cuda/requirements.txt b/test/test_cuda/requirements.txtindex fa36ffcc4..3d4e3e101 100644--- a/test/test_cuda/requirements.txt+++ b/test/test_cuda/requirements.txt@@ -1,11 +1,12 @@# pip install -v git+https://github.com/casper-hansen/AutoAWQ.git --no-build-isolation+--extra-index-url https://wheels.vllm.ai/nightlyauto-gptqeinops# pip install -v git+https://github.com/ModelCloud/GPTQModel.git --no-build-isolation-lm-eval>=0.4.9.1+lm_eval @ git+https://github.com/EleutherAI/lm-evaluation-harness.git@mainoptimumpandaspillowtorchvisionnumba-vllm>=0.8.5.post1+vllm\ No newline at end of filediff --git a/test/test_cuda/requirements_sglang.txt b/test/test_cuda/requirements_sglang.txtindex db0e61c5d..384c162b5 100644--- a/test/test_cuda/requirements_sglang.txt+++ b/test/test_cuda/requirements_sglang.txt@@ -1 +1 @@-sglang>=0.5.4.post2+sglang @ git+https://github.com/sgl-project/sglang.git@main#subdirectory=python\ No newline at end of file