PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度
add qwen35 ut (#1482)
Wenhua Cheng2b5d1175e783
diff --git a/test/helpers.py b/test/helpers.pyindex 2f98e4e55..207864ccf 100644--- a/test/helpers.py+++ b/test/helpers.py@@ -183,6 +183,16 @@ def is_pytest_mode_lazy():return pytest.mode == "lazy"+def check_version(lib):+ try:+ from transformers.utils.versions import require_version++ require_version(lib)+ return True+ except Exception:+ return False++# General model inference codedef model_infer(model, tokenizer, apply_chat_template=False):"""Run model inference and print generated outputs."""diff --git a/test/test_cuda/models/test_moe_model.py b/test/test_cuda/models/test_moe_model.pyindex 40c545015..8c275d595 100644--- a/test/test_cuda/models/test_moe_model.py+++ b/test/test_cuda/models/test_moe_model.py@@ -11,6 +11,8 @@ from transformers.models.qwen3_vl_moe.modeling_qwen3_vl_moe import Qwen3VLMoeForfrom auto_round import AutoRoundfrom auto_round.modeling.fused_moe.replace_modules import ReplacementModuleBase+from ...helpers import check_version+@pytest.fixturedef setup_gpt_oss():@@ -50,7 +52,7 @@ def setup_qwen3_vl_moe():# TODO: Remove after https://github.com/huggingface/transformers/pull/43453 is mergedconfig.text_config.pad_token_id = None-+ # Reduce model depth for faster and cheaper test runsconfig.vision_config.num_hidden_layers = 1config.text_config.num_hidden_layers = 1config.num_hidden_layers = 1 # Reduce layers for testing@@ -174,6 +176,54 @@ def setup_qwen3():return model, tokenizer, output_dir, config+def setup_qwen35_moe():+ """Fixture to set up the Qwen3.5 MoE model, tokenizer, and processor."""+ from transformers import Qwen3_5MoeForConditionalGeneration++ model_name = "/models/Qwen3.5-35B-A3B"+ tokenizer = AutoTokenizer.from_pretrained(model_name)+ config = AutoConfig.from_pretrained(model_name)+ config.text_config.pad_token_id = None+ config.vision_config.num_hidden_layers = 1+ config.text_config.num_hidden_layers = 4+ config.num_hidden_layers = 1 # Reduce layers for testing+ config.text_config.layer_types = config.text_config.layer_types[+ : config.text_config.num_hidden_layers+ ] # Reduce layers for testing+ config.text_config.use_cache = False+ processor = AutoProcessor.from_pretrained(model_name)+ model = Qwen3_5MoeForConditionalGeneration(config)+ # model = Qwen3_5MoeForConditionalGeneration.from_pretrained(model_name)+ output_dir = "test_quantized_qwen35_moe"+ return model, tokenizer, processor, output_dir, config++[email protected](not check_version("transformers>=5.2.0"), reason="requires transformers >= 5.2.0")+def test_qwen3_5_moe(setup_qwen35_moe):+ model, tokenizer, processor, output_dir, config = setup_qwen35_moe+ ar = AutoRound(+ model,+ tokenizer=tokenizer,+ processor=processor,+ nsamples=2,+ seqlen=32,+ iters=1,+ )+ quantized_model, _ = ar.quantize_and_save(format="auto_round", output_dir=output_dir)+ assert quantized_model is not None, "Quantized model should not be None."+ from transformers import Qwen3_5MoeForConditionalGeneration++ loaded_model = Qwen3_5MoeForConditionalGeneration.from_pretrained(output_dir)+ loaded_model.to("cuda")++ inp = torch.randint(0, 100, (1, 64)).to("cuda")+ with torch.inference_mode():+ loaded_out = loaded_model(inp)++ shutil.rmtree(output_dir, ignore_errors=True)++def test_qwen3_vl_moe_mxfp(setup_qwen3_vl_moe):model, tokenizer, processor, output_dir, config = setup_qwen3_vl_moeautoround = AutoRound(