PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度
fix critical bug of get_multimodal_block_names (#509)
Heng Guoee890b2cf1a2
diff --git a/README.md b/README.mdindex 37e4872d9..33048db91 100644--- a/README.md+++ b/README.md@@ -224,7 +224,7 @@ autoround.quantize_and_save(output_dir, format='auto_round')</details>### API Usage for VLMs-+If you encounter issues during quantization, try setting iters=0 (to enable RTN) and use group_size=32 for better results.<details><summary>Click to expand</summary>diff --git a/auto_round/export/export_to_awq/export.py b/auto_round/export/export_to_awq/export.pyindex 4759ae01e..a55c4f863 100644--- a/auto_round/export/export_to_awq/export.py+++ b/auto_round/export/export_to_awq/export.py@@ -27,7 +27,7 @@ import torch.nn as nnfrom auto_round.utils import (logger, get_module,set_module,check_to_quantized,- get_multimodal_block_names,+ get_block_names,extract_block_names_to_str, supported_layer_types)import copyimport json@@ -84,7 +84,7 @@ def save_quantized_as_autoawq(output_dir, inplace=True, **kwargs):if processor is not None:processor.save_pretrained(output_dir)# mllm models- all_blocks = get_multimodal_block_names(model, quant_vision=True)+ all_blocks = get_block_names(model, quant_vision=True)all_block_names = extract_block_names_to_str(all_blocks)all_block_names = all_block_names.split(',')to_quant_block_names = to_quant_block_names.split(',')diff --git a/auto_round/inference/convert_model.py b/auto_round/inference/convert_model.pyindex 67a8b7310..417276939 100644--- a/auto_round/inference/convert_model.py+++ b/auto_round/inference/convert_model.py@@ -21,11 +21,13 @@ import torch.nn as nnfrom transformers.pytorch_utils import Conv1D-from auto_round.utils import (get_module, set_module, is_hpu_supported, get_multimodal_block_names,- find_matching_blocks, get_layer_names_in_block, check_to_quantized)+from auto_round.utils import (+ get_module, set_module, is_hpu_supported, get_block_names, find_matching_blocks,+ get_layer_names_in_block, check_to_quantized)-from auto_round.inference.backend import (get_layer_backend, dynamic_import_inference_linear,- find_backend, BackendInfos, get_highest_priority_backend, process_requirement)+from auto_round.inference.backend import (+ get_layer_backend, dynamic_import_inference_linear, find_backend, BackendInfos, get_highest_priority_backend,+ process_requirement)logger = getLogger(__name__)@@ -231,7 +233,7 @@ def get_layer_config(model, quantization_config):]else:# Find matching blocks if no explicit names are provided- all_blocks = get_multimodal_block_names(model, quant_vision=True)+ all_blocks = get_block_names(model, quant_vision=True)quant_block_list = find_matching_blocks(model, all_blocks, to_quant_block_names)# Get layer names that will be quantizeddiff --git a/auto_round/mllm/autoround_mllm.py b/auto_round/mllm/autoround_mllm.pyindex 1427df4d7..e0ba63586 100644--- a/auto_round/mllm/autoround_mllm.py+++ b/auto_round/mllm/autoround_mllm.py@@ -23,7 +23,7 @@ from ..utils import (detect_device,to_device,to_dtype,- get_multimodal_block_names,+ get_block_names,find_matching_blocks,extract_block_names_to_str,clear_memory@@ -163,7 +163,7 @@ class AutoRoundMLLM(AutoRound):enable_torch_compile: bool = False,**kwargs,):- all_blocks = get_multimodal_block_names(model, quant_nontext_module)+ all_blocks = get_block_names(model, quant_nontext_module)self.quant_block_list = find_matching_blocks(model, all_blocks, to_quant_block_names)if to_quant_block_names is None:to_quant_block_names = extract_block_names_to_str(self.quant_block_list)diff --git a/auto_round/script/mllm.py b/auto_round/script/mllm.pyindex 6fb6aa1b4..4c74a1732 100644--- a/auto_round/script/mllm.py+++ b/auto_round/script/mllm.py@@ -145,7 +145,7 @@ class BasicArgumentParser(argparse.ArgumentParser):choices=["fp16", "float16", "bf16", "bfloat16", "fp32", "float32"],help="force to convert the dtype, some backends supports fp16 dtype better")- self.add_argument("--act_bits", default=32, type=int, help="activation bits")+ self.add_argument("--act_bits", default=16, type=int, help="activation bits")self.add_argument("--fp_layers", default="", type=str, help="layers to maintain original data type")diff --git a/auto_round/utils.py b/auto_round/utils.pyindex 4a0c82b8e..9e1b64511 100644--- a/auto_round/utils.py+++ b/auto_round/utils.py@@ -380,7 +380,7 @@ def find_matching_blocks(model, all_blocks, to_quant_block_names):return target_blocks-def get_block_names(model):+def get_block_names(model, quant_vision=False):"""Get the block names for transformers-like networks.Args:@@ -389,43 +389,43 @@ def get_block_names(model):Returns:block_names: A list whose elements are list of block's layer names"""- block_names = []- target_modules = []- for n, m in model.named_modules():- if hasattr(type(m), "__name__") and "ModuleList" in type(m).__name__:- target_modules.append((n, m))- break ## only find the first modulelist, may be not robust- for i, target_m in enumerate(target_modules):- block_names.append([])- for n, m in target_m[1].named_children():- block_names[i].append(target_m[0] + "." + n)- return block_names---def get_multimodal_block_names(model, quant_vision=False):- """Get the multimodal model block names for transformers-like networks.-- Args:- model: The model.-- Returns:- block_names: A list whose elements are list of block's layer names- """- if hasattr(model, "config") and model.config.model_type in SPECIAL_MULTIMODAL_BLOCK.keys():- return SPECIAL_MULTIMODAL_BLOCK.get(model.config.model_type)(model, quant_vision=quant_vision)- block_names = []- target_modules = []- vison_blocks_tuple = ("vision", "visual",)- for n, m in model.named_modules():- if hasattr(type(m), "__name__") and "ModuleList" in type(m).__name__:- if quant_vision or all(key not in n.lower() for key in (vison_blocks_tuple)):+ def _get_llm_block_names(model):+ block_names = []+ target_modules = []+ for n, m in model.named_modules():+ if hasattr(type(m), "__name__") and "ModuleList" in type(m).__name__:target_modules.append((n, m))- validate_modules(target_modules, quant_vision, vison_blocks_tuple)- for i, target_m in enumerate(target_modules):- block_names.append([])- for n, m in target_m[1].named_children():- block_names[i].append(target_m[0] + "." + n)- return block_names+ break ## only find the first modulelist, may be not robust+ for i, target_m in enumerate(target_modules):+ block_names.append([])+ for n, m in target_m[1].named_children():+ block_names[i].append(target_m[0] + "." + n)+ return block_names++ def _get_vlm_block_names(model, quant_vision=False):+ if hasattr(model, "config") and model.config.model_type in SPECIAL_MULTIMODAL_BLOCK.keys():+ return SPECIAL_MULTIMODAL_BLOCK.get(model.config.model_type)(model, quant_vision=quant_vision)+ block_names = []+ target_modules = []+ vision_blocks_tuple = ("vision", "visual", "image", "img")+ last_block_name = ""+ for n, m in model.named_modules():+ if hasattr(type(m), "__name__") and "ModuleList" in type(m).__name__:+ if quant_vision or all(key not in n.lower() for key in (vision_blocks_tuple)):+ if last_block_name and last_block_name in n:+ continue+ target_modules.append((n, m))+ last_block_name = n+ for i, target_m in enumerate(target_modules):+ block_names.append([])+ for n, m in target_m[1].named_children():+ block_names[i].append(target_m[0] + "." + n)+ return block_names++ if quant_vision or not is_pure_text_model(model):+ return _get_vlm_block_names(model, quant_vision=quant_vision)+ else:+ return _get_llm_block_names(model)def collect_best_params(block):@@ -1265,3 +1265,19 @@ def _gguf_args_check(args):return args+def is_pure_text_model(model):+ """verify on: phi-3.5, Mistral-Small-3.1, gemma-3, qwen2-vl, """+ if hasattr(model,"config") and hasattr(model.config, "vision_config"):+ return False+ if hasattr(model.__class__, "main_input_name") and model.__class__.main_input_name != "input_ids":+ return False+ for module in model.modules():+ if hasattr(module.__class__, "main_input_name") and module.__class__.main_input_name != "input_ids":+ return False+ if "vision" in str(module.__class__).lower():+ return False+ if "image" in str(module.__class__).lower():+ return False+ if "img" in str(module.__class__).lower():+ return False+ return Truediff --git a/test/test_autoopt.py b/test/test_autoopt.pyindex 6dcb65543..3ea4565d7 100644--- a/test/test_autoopt.py+++ b/test/test_autoopt.py@@ -51,8 +51,8 @@ class TestAutoRound(unittest.TestCase):def test_Adam(self):bits, group_size, sym = 4, 128, False- from auto_round.utils import get_multimodal_block_names- llm_block_names = get_multimodal_block_names(self.model, quant_vision=True)+ from auto_round.utils import get_block_names+ llm_block_names = get_block_names(self.model, quant_vision=True)bits, group_size, sym, batch_size = 4, 128, False, 20adamround = AutoRoundAdam(self.model,diff --git a/test/test_block_names.py b/test/test_block_names.pyindex 0a106de6c..c98885e50 100644--- a/test/test_block_names.py+++ b/test/test_block_names.py@@ -1,3 +1,4 @@+import osimport shutilimport sysimport unittest@@ -5,7 +6,7 @@ sys.path.insert(0, ".")sys.path.insert(0, "..")import torchimport torch.nn as nn-from transformers import AutoModelForCausalLM, AutoTokenizer+from transformers import AutoModelForCausalLM, AutoTokenizer, AutoConfigfrom auto_round import AutoRoundclass LLMDataLoader:@@ -119,14 +120,11 @@ class TestQuantizationBlocks(unittest.TestCase):experts_per_group = 2self.llm_dataloader = LLMDataLoader(input_size)self.model = NestedMoEModel(input_size, hidden_size, num_groups, experts_per_group)- from auto_round.utils import get_multimodal_block_names, get_block_names+ from auto_round.utils import get_block_namesllm_block_names = get_block_names(self.model)all_block_names = []- try:- all_block_names = get_multimodal_block_names(self.model, quant_vision=True)- except:- pass- assert len(llm_block_names) != len(all_block_names)+ all_block_names = get_block_names(self.model, quant_vision=True)+ assert len(llm_block_names) == len(all_block_names)def test_multimodal_quant(self):@@ -136,10 +134,10 @@ class TestQuantizationBlocks(unittest.TestCase):text_input_size = 10hidden_size = 10self.model = SimpleMultimodalModel(text_input_size, image_input_size, hidden_size, num_text_encoders, num_image_encoders)- from auto_round.utils import get_multimodal_block_names, get_block_names+ from auto_round.utils import get_block_namesllm_block_names = get_block_names(self.model)- block_names_wo_vision = get_multimodal_block_names(self.model, quant_vision=False)- block_names_with_vision = get_multimodal_block_names(self.model, quant_vision=True)+ block_names_wo_vision = get_block_names(self.model, quant_vision=False)+ block_names_with_vision = get_block_names(self.model, quant_vision=True)assert block_names_wo_vision == llm_block_namesassert len(block_names_wo_vision) != (block_names_with_vision)@@ -163,25 +161,50 @@ class TestQuantizationBlocks(unittest.TestCase):to_quant_block_names=llm_block_names)autoround.quantize()- try:- import auto_gptq- except:- return- if not torch.cuda.is_available():- return+quantized_model_path = "./saved"autoround.save_quantized(quantized_model_path, inplace=False, safe_serialization=False, format="auto_round")-+ from auto_round import AutoRoundConfigmodel = AutoModelForCausalLM.from_pretrained(quantized_model_path, device_map="auto")tokenizer = AutoTokenizer.from_pretrained(quantized_model_path)text = "There is a girl who likes adventure,"inputs = tokenizer(text, return_tensors="pt").to(model.device)print(tokenizer.decode(model.generate(**inputs, max_new_tokens=50)[0]))- shutil.rmtree("./saved", ignore_errors=True)quant_config = model.config.quantization_configassert quant_config.to_quant_block_names is not None+ shutil.rmtree("./saved", ignore_errors=True)-+ def test_moe(self):+ from auto_round.utils import get_block_names+ model_name = "Qwen/Qwen1.5-MoE-A2.7B"+ # config = AutoConfig.from_pretrained(model_name)+ model = AutoModelForCausalLM.from_pretrained(model_name)+++ block_name = get_block_names(model)+ block_name_2 = get_block_names(model, quant_vision=True)+ self.assertTrue(block_name == block_name_2)+ self.assertTrue(len(block_name_2) == 1)+ self.assertTrue('model.layers.23' == block_name_2[0][-1])+ ##tokenizer = AutoTokenizer.from_pretrained(model_name)+ # python_path = sys.executable+ # res = os.system(+ # f"cd .. && CUDA_VISIBLE_DEVICES=0 {python_path} -m auto_round --model {model_name} --iter 1 --nsamples 1 --format auto_round --output_dir test/saved --disable_eval")+ # if res > 0 or res == -1:+ # assert False, "cmd line test fail, please have a check"+ #+ # quantized_model_path = "../saved"+ #+ # from auto_round import AutoHfQuantizer+ # model = AutoModelForCausalLM.from_pretrained(quantized_model_path, device_map="auto")+ # tokenizer = AutoTokenizer.from_pretrained(quantized_model_path)+ # text = "There is a girl who likes adventure,"+ # inputs = tokenizer(text, return_tensors="pt").to(model.device)+ # print(tokenizer.decode(model.generate(**inputs, max_new_tokens=50)[0]))+ # shutil.rmtree("./saved", ignore_errors=True)+ # quant_config = model.config.quantization_config+ # assert quant_config.to_quant_block_names is not None+if __name__ == "__main__":diff --git a/test/test_mllm.py b/test/test_mllm.pyindex 2845c1392..68b27752e 100644--- a/test/test_mllm.py+++ b/test/test_mllm.py@@ -69,14 +69,14 @@ class TestAutoRoundMLLM(unittest.TestCase):autoround.save_quantized("./saved/", format="auto_round", inplace=True)def test_quant_block_names(self):- from auto_round.utils import get_multimodal_block_names,find_matching_blocks+ from auto_round.utils import get_block_names,find_matching_blockstokenizer = AutoTokenizer.from_pretrained(self.model_name)processor = AutoProcessor.from_pretrained(self.model_name, trust_remote_code=True)model = Qwen2VLForConditionalGeneration.from_pretrained(self.model_name, trust_remote_code=True, device_map="auto")to_quant_block_names = 'visual.*12,layers.0,model.layers.*9'target_blocks = [['visual.blocks.12'], ['model.layers.0', 'model.layers.9', 'model.layers.19']]- all_blocks = get_multimodal_block_names(model, quant_vision=True)+ all_blocks = get_block_names(model, quant_vision=True)blocks = find_matching_blocks(model, all_blocks, to_quant_block_names)assert target_blocks == blocks@@ -102,6 +102,16 @@ class TestAutoRoundMLLM(unittest.TestCase):nsamples=2,batch_size=1, iters=2, dataset=dataset, seqlen=1)autoround.quantize()++ def test_pure_text_model_check(self):+ from transformers import AutoModelForCausalLM+ from auto_round.utils import is_pure_text_model+ model = Qwen2VLForConditionalGeneration.from_pretrained(+ self.model_name, trust_remote_code=True, device_map="auto")+ self.assertFalse(is_pure_text_model(model))+ model = AutoModelForCausalLM.from_pretrained("facebook/opt-125m", trust_remote_code=True)+ self.assertTrue(is_pure_text_model(model))+if __name__ == "__main__":unittest.main()diff --git a/test_cuda/test_get_block_name.py b/test_cuda/test_get_block_name.pynew file mode 100644index 000000000..f077a65d6--- /dev/null+++ b/test_cuda/test_get_block_name.py@@ -0,0 +1,175 @@+import copy+import shutil+import sys+import unittest++sys.path.insert(0, "..")+import torch+import transformers+from transformers import AutoModelForCausalLM, AutoTokenizer, Qwen2VLForConditionalGeneration, AutoModelForVision2Seq, \+ Gemma3ForConditionalGeneration, Mistral3ForConditionalGeneration+from auto_round.utils import get_block_names, is_pure_text_model+from auto_round import AutoRound+++class TestAutoRound(unittest.TestCase):+ @classmethod+ def setUpClass(self):+ pass++ @classmethod+ def tearDownClass(self):+ shutil.rmtree("runs", ignore_errors=True)++ def check_block_names(self, block_names, prefixs=[], n_layers=[]):+ for i, block_name in enumerate(block_names):+ prefix = prefixs[i]+ n_layer = n_layers[i]+ expected_block_names = [prefix + "." + str(i) for i in range(n_layer)]+ assert block_name == expected_block_names++ def test_glm4(self):+ model_name = "/models/glm-4-9b-chat"+ model = AutoModelForCausalLM.from_pretrained(model_name, torch_dtype="auto", trust_remote_code=True)+ block_names = get_block_names(model)+ self.check_block_names(block_names, ["transformer.encoder.layers"], [40])+ assert is_pure_text_model(model) == True++ def test_opt_125m(self):+ model_name = "/models/opt-125m"+ model = AutoModelForCausalLM.from_pretrained(model_name, torch_dtype="auto", trust_remote_code=True)+ block_names = get_block_names(model)+ self.check_block_names(block_names, ["model.decoder.layers"], [12])++ assert is_pure_text_model(model) == True++ def test_Qwen(self):+ model_name = "/models/Qwen2.5-7B-Instruct"+ model = AutoModelForCausalLM.from_pretrained(model_name, torch_dtype="auto", trust_remote_code=True)+ block_names = get_block_names(model)+ self.check_block_names(block_names, ["model.layers"], [28])+ assert is_pure_text_model(model) == True++ def test_phi4(self):+ model_name = "/models/phi-4"+ model = AutoModelForCausalLM.from_pretrained(model_name, torch_dtype="auto", trust_remote_code=True)+ block_names = get_block_names(model)+ self.check_block_names(block_names, ["model.layers"], [40])+ assert is_pure_text_model(model) == True++ def test_llama3(self):+ model_name = "/models/Meta-Llama-3.1-8B-Instruct"+ model = AutoModelForCausalLM.from_pretrained(model_name, torch_dtype="auto", trust_remote_code=True)+ block_names = get_block_names(model)+ self.check_block_names(block_names, ["model.layers"], [32])+ assert is_pure_text_model(model) == True++ def test_mixtral(self):+ model_name = "/models/Mixtral-8x7B-Instruct-v0.1"+ model = AutoModelForCausalLM.from_pretrained(model_name, torch_dtype="auto", trust_remote_code=True)+ block_names = get_block_names(model)+ self.check_block_names(block_names, ["model.layers"], [32])+ assert is_pure_text_model(model) == True++ def test_falcon(self):+ model_name = "/models/Falcon3-7B-Instruct"+ model = AutoModelForCausalLM.from_pretrained(model_name, torch_dtype="auto", trust_remote_code=True)+ block_names = get_block_names(model)+ self.check_block_names(block_names, ["model.layers"], [28])+ assert is_pure_text_model(model) == True++ def test_orca(self):+ model_name = "/models/Orca-2-7b"+ model = AutoModelForCausalLM.from_pretrained(model_name, torch_dtype="auto", trust_remote_code=True)+ block_names = get_block_names(model)+ self.check_block_names(block_names, ["model.layers"], [32])+ assert is_pure_text_model(model) == True++ def test_OLMo(self):+ model_name = "/models/OLMo-2-1124-7B-Instruct"+ model = AutoModelForCausalLM.from_pretrained(model_name, torch_dtype="auto", trust_remote_code=True)+ block_names = get_block_names(model)+ self.check_block_names(block_names, ["model.layers"], [32])+ assert is_pure_text_model(model) == True++ def test_Qwen2VL(self):+ model_name = "/models/Qwen2-VL-2B-Instruct"+ model = Qwen2VLForConditionalGeneration.from_pretrained(model_name, torch_dtype="auto", trust_remote_code=True)+ block_names = get_block_names(model)+ self.check_block_names(block_names, ["model.layers"], [28])++ block_names = get_block_names(model, quant_vision=True)+ self.check_block_names(block_names, ["visual.blocks", "model.layers"], [32, 28])+ assert is_pure_text_model(model) == False++ def test_Llama32(self):+ model_name = "/models/Llama-3.2-11B-Vision-Instruct"+ model = AutoModelForVision2Seq.from_pretrained(model_name, torch_dtype="auto", trust_remote_code=True)+ block_names = get_block_names(model)+ self.check_block_names(block_names, ["language_model.model.layers"], [40])++ block_names = get_block_names(model, quant_vision=True)+ self.check_block_names(block_names,+ ["vision_model.transformer.layers", "vision_model.global_transformer.layers",+ "language_model.model.layers"], [32, 8, 40])++ assert is_pure_text_model(model) == False++ def test_SmolVLM(self):+ model_name = "/models/SmolVLM-Instruct"+ model = AutoModelForVision2Seq.from_pretrained(model_name, torch_dtype="auto", trust_remote_code=True)+ block_names = get_block_names(model)+ self.check_block_names(block_names, ["model.text_model.layers"], [24])++ block_names = get_block_names(model, quant_vision=True)+ self.check_block_names(block_names, ["model.vision_model.encoder.layers", "model.text_model.layers"], [27, 24])+ assert is_pure_text_model(model) == False++ def test_glm_4v(self):+ model_name = "/models/glm-4v-9b"+ model = AutoModelForCausalLM.from_pretrained(model_name, torch_dtype="auto", trust_remote_code=True)+ block_names = get_block_names(model)+ self.check_block_names(block_names, ["transformer.encoder.layers"], [40])++ block_names = get_block_names(model, quant_vision=True)+ self.check_block_names(block_names, ["transformer.encoder.layers", "transformer.vision.transformer.layers"],+ [40, 63])+ assert is_pure_text_model(model) == False++ def test_gemma3(self):+ model_name = "/models/gemma-3-12b-it"+ model = Gemma3ForConditionalGeneration.from_pretrained(model_name, torch_dtype="auto", trust_remote_code=True)+ block_names = get_block_names(model)+ self.check_block_names(block_names, ["language_model.model.layers"], [48])++ block_names = get_block_names(model, quant_vision=True)+ self.check_block_names(block_names, ["vision_tower.vision_model.encoder.layers", "language_model.model.layers"],+ [27, 48])+ assert is_pure_text_model(model) == False++ def test_Mistral3(self):+ model_name = "/models/Mistral-Small-3.1-24B-Instruct-2503"+ model = Mistral3ForConditionalGeneration.from_pretrained(model_name, torch_dtype="auto", trust_remote_code=True)+ block_names = get_block_names(model)+ self.check_block_names(block_names, ["language_model.model.layers"], [40])++ block_names = get_block_names(model, quant_vision=True)+ self.check_block_names(block_names, ["vision_tower.transformer.layers", "language_model.model.layers"],+ [24, 40])+ assert is_pure_text_model(model) == False++ def test_Molmo(self):+ model_name = "/models/Molmo-7B-D-0924"+ model = AutoModelForCausalLM.from_pretrained(model_name, torch_dtype="auto", trust_remote_code=True)+ block_names = get_block_names(model)+ self.check_block_names(block_names, ["model.transformer.blocks"], [28])++ block_names = get_block_names(model, quant_vision=True)+ self.check_block_names(block_names,+ ["model.transformer.blocks", "model.vision_backbone.image_vit.transformer.resblocks"],+ [28, 23])+ assert is_pure_text_model(model) == False+++if __name__ == "__main__":+ unittest.main()