PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度
support awq format for vlms (#398)
WeiweiZhang1c82aa88ae2b8
diff --git a/auto_round/export/export_to_awq/export.py b/auto_round/export/export_to_awq/export.pyindex 1921de446..b13d5d756 100644--- a/auto_round/export/export_to_awq/export.py+++ b/auto_round/export/export_to_awq/export.py@@ -24,7 +24,11 @@ import osimport torchimport torch.nn as nn-from auto_round.utils import logger, get_module, set_module, check_to_quantized+from auto_round.utils import (logger, get_module,+ set_module,+ check_to_quantized,+ get_multimodal_block_names,+ extract_block_names_to_str)import copyimport jsonfrom .utils import WQLinear_GEMM, clear_memory@@ -69,15 +73,22 @@ def save_quantized_as_autoawq(output_dir, inplace=True, **kwargs):"""Export the model to autogptq format to easily leverage cuda kernel."""model = kwargs["model"]layer_config = kwargs["layer_config"]-+ to_quant_block_names = kwargs.get("to_quant_block_names", None)tokenizer = kwargs.get("tokenizer", None)processor = kwargs.get("processor", None)-+ modules_to_not_convert = []+logger.info("Saving quantized model to auto_awq format")if tokenizer is not None:tokenizer.save_pretrained(output_dir)if processor is not None:processor.save_pretrained(output_dir)+ # mllm models+ all_blocks = get_multimodal_block_names(model, quant_vision=True)+ all_block_names = extract_block_names_to_str(all_blocks)+ all_block_names = all_block_names.split(',')+ to_quant_block_names = to_quant_block_names.split(',')+ modules_to_not_convert = list(set(all_block_names) - set(to_quant_block_names))if inplace:compressed_model = model.to("cpu")@@ -101,11 +112,13 @@ def save_quantized_as_autoawq(output_dir, inplace=True, **kwargs):if output_dir is None:return compressed_model- modules_to_not_convert = []+layer_config = kwargs["layer_config"]for key in layer_config.keys():- if not check_to_quantized(layer_config[key]):+ if not check_to_quantized(layer_config[key]) and \+ not any(name in key for name in modules_to_not_convert):modules_to_not_convert.append(key)+quantization_config["quant_method"] = "awq"quantization_config["zero_point"] = not quantization_config["sym"]quantization_config["version"] = "gemm"@@ -115,7 +128,8 @@ def save_quantized_as_autoawq(output_dir, inplace=True, **kwargs):if hasattr(compressed_model, "config"):compressed_model.config.quantization_config = quantization_config- save(compressed_model, output_dir, safe_serialization=True)+ safe_serialization = kwargs.get('safe_serialization', True)+ save(compressed_model, output_dir, safe_serialization=safe_serialization)return compressed_model@@ -146,3 +160,4 @@ def save(model: nn.Module, save_dir: str, max_shard_size: str = "5GB", safe_seriif hasattr(model, "config") and hasattr(model.config, "quantization_config"):with open(os.path.join(save_dir, config_file), "w", encoding="utf-8") as f:json.dump(model.config.quantization_config, f, indent=2)+diff --git a/auto_round/script/mllm.py b/auto_round/script/mllm.pyindex 7036ec5b0..a3a3dd9fc 100644--- a/auto_round/script/mllm.py+++ b/auto_round/script/mllm.py@@ -233,7 +233,7 @@ def setup_lmeval_parser():def tune(args):if args.format is None:args.format = "auto_round"- supported_formats = ["auto_round", "auto_round:auto_gptq", "auto_round:auto_awq"]+ supported_formats = ["auto_round", "auto_round:auto_gptq", "auto_round:auto_awq", "auto_awq"]if not args.quant_nontext_module:supported_formats.extend(["auto_gptq", "auto_gptq:marlin"])@@ -392,6 +392,12 @@ def tune(args):if "--truncation" not in sys.argv:args.truncation = None++ if "auto_awq" in args.format:+ from auto_round.utils import check_awq_gemm_compatibility+ awq_supported, info = check_awq_gemm_compatibility(model,args.bits,args.group_size, not args.asym, layer_config)+ if not awq_supported:+ logger.warning(f"The AutoAWQ format may not be supported due to {info}")autoround = round(model, tokenizer, processor=processor, image_processor=image_processor, dataset=args.dataset,extra_data_dir=args.extra_data_dir, bits=args.bits, group_size=args.group_size,@@ -512,3 +518,4 @@ def lmms_eval(args):apply_chat_template=False,)return results+diff --git a/test_cuda/test_support_vlms.py b/test_cuda/test_support_vlms.pyindex 91e69aa3d..357fee952 100644--- a/test_cuda/test_support_vlms.py+++ b/test_cuda/test_support_vlms.py@@ -81,7 +81,7 @@ class TestSupportVLMS(unittest.TestCase):## test tuneres = os.system(f"cd .. && {self.python_path} -m auto_round --mllm "- f"--model {model_path} --iter 2 --output_dir {self.save_dir} --device {self.device}")+ f"--model {model_path} --iter 2 --output_dir {self.save_dir} --device {self.device}")self.assertFalse(res > 0 or res == -1, msg="Phi-3.5 tuning fail")## test infer@@ -114,11 +114,74 @@ class TestSupportVLMS(unittest.TestCase):image_inputs = Image.open(requests.get(image_url, stream=True).raw)inputs = processor(prompt, image_inputs, return_tensors="pt").to(model.device)- generation_args = {+ generation_args = {+ "max_new_tokens": 1000,+ "temperature": 0.0,+ "do_sample": False,+ }++ generate_ids = model.generate(**inputs,+ eos_token_id=processor.tokenizer.eos_token_id,+ **generation_args+ )++ # remove input tokens+ generate_ids = generate_ids[:, inputs['input_ids'].shape[1]:]+ response = processor.batch_decode(generate_ids,+ skip_special_tokens=True,+ clean_up_tokenization_spaces=False)[0]+ print(response)+ shutil.rmtree(quantized_model_path, ignore_errors=True)++ def test_phi3_vision_awq(self):+ model_path = "/models/Phi-3.5-vision-instruct/"+ ## test tune+ res = os.system(+ f"cd .. && {self.python_path} -m auto_round --mllm "+ f"--model {model_path} --iter 2 --quant_nontext_module "+ f"--nsample 64 --seqlen 32 "+ f"--format auto_awq --output_dir {self.save_dir} --device {self.device}")+ self.assertFalse(res > 0 or res == -1, msg="Phi-3.5 tuning fail")++ ## test infer+ from transformers import AutoModelForCausalLM, AutoProcessor+ from auto_round.export.export_to_awq import WQLinear_GEMM+ quantized_model_path = os.path.join(self.save_dir, "Phi-3.5-vision-instruct-w4g128-auto_awq")+ res = os.system(f"cp /models/Phi-3.5-vision-instruct/*.py {quantized_model_path}")+ model = AutoModelForCausalLM.from_pretrained(+ quantized_model_path,+ device_map=f"cuda:{self.device}",+ trust_remote_code=True,+ torch_dtype="auto"+ )+ assert "WQLinear_GEMM" in str(+ type(model.model.vision_embed_tokens.img_processor.vision_model.encoder.layers[0].mlp.fc1)), \+ "model quantization failed."+ processor = AutoProcessor.from_pretrained(quantized_model_path,+ trust_remote_code=True,+ num_crops=4+ )++ image_url = "https://qianwen-res.oss-cn-beijing.aliyuncs.com/Qwen-VL/assets/demo.jpeg"+ content = "Describe this image."+ messages = [+ {"role": "user",+ "content": "<|image_1|>\n"+content},+ ]++ prompt = processor.tokenizer.apply_chat_template(+ messages,+ tokenize=False,+ add_generation_prompt=True+ )+ image_inputs = Image.open(requests.get(image_url, stream=True).raw)+ inputs = processor(prompt, image_inputs, return_tensors="pt").to(model.device)++ generation_args = {"max_new_tokens": 1000,"temperature": 0.0,"do_sample": False,- }+ }generate_ids = model.generate(**inputs,eos_token_id=processor.tokenizer.eos_token_id,@@ -272,4 +335,4 @@ class TestSupportVLMS(unittest.TestCase):shutil.rmtree(self.save_dir, ignore_errors=True)if __name__ == "__main__":- unittest.main()\ No newline at end of file+ unittest.main()