PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度
Add gpu ut (#370)
Wenhua Chengecc17be01b00
diff --git a/test_cuda/test_2_3bits.py b/test_cuda/test_2_3bits.pynew file mode 100644index 000000000..361b87e67--- /dev/null+++ b/test_cuda/test_2_3bits.py@@ -0,0 +1,104 @@+import copy+import shutil+import sys+import unittest+import re++sys.path.insert(0, "..")+import torch+import transformers+from transformers import AutoModelForCausalLM, AutoTokenizer++from auto_round import AutoRound+from auto_round.eval.evaluation import simple_evaluate+from lm_eval.utils import make_table # pylint: disable=E0401+++def get_accuracy(data):+ match = re.search(r'\|acc\s+\|[↑↓]\s+\|\s+([\d.]+)\|', data)++ if match:+ accuracy = float(match.group(1))+ return accuracy+ else:+ return 0.0+++class TestAutoRound(unittest.TestCase):+ @classmethod+ def setUpClass(self):+ self.save_dir = "./saved"+ self.tasks = "lambada_openai"++ @classmethod+ def tearDownClass(self):+ shutil.rmtree("./saved", ignore_errors=True)+ shutil.rmtree("runs", ignore_errors=True)++ def test_3bits_autogptq(self):+ model_name = "/models/opt-125m"+ model = AutoModelForCausalLM.from_pretrained(model_name, torch_dtype=torch.float16, device_map="auto")+ tokenizer = AutoTokenizer.from_pretrained(model_name)+ autoround = AutoRound(model, tokenizer, bits=3)+ autoround.quantize()++ autoround.save_quantized(self.save_dir, format="auto_gptq", inplace=False)+ model_args = f"pretrained={self.save_dir}"+ res = simple_evaluate(model="hf", model_args=model_args,+ tasks=self.tasks,+ batch_size="auto")+ res = make_table(res)++ accuracy = get_accuracy(res)+ assert accuracy > 0.30+ shutil.rmtree("./saved", ignore_errors=True)++ def test_norm_bias_tuning(self):+ model_name = "/models/opt-125m"+ model = AutoModelForCausalLM.from_pretrained(model_name, torch_dtype=torch.float16, device_map="auto")+ tokenizer = AutoTokenizer.from_pretrained(model_name)+ autoround = AutoRound(model, tokenizer, bits=2, group_size=64, enable_norm_bias_tuning=True)+ autoround.quantize()++ ##test auto_round format+ autoround.save_quantized(self.save_dir, format="auto_round", inplace=False)+ model_args = f"pretrained={self.save_dir}"+ res = simple_evaluate(model="hf", model_args=model_args,+ tasks=self.tasks,+ batch_size="auto")+ res = make_table(res) ##0.2212+ accuracy = get_accuracy(res)+ assert accuracy > 0.20+ shutil.rmtree("./saved", ignore_errors=True)++ def test_2bits_autoround(self):+ model_name = "/models/opt-125m"+ model = AutoModelForCausalLM.from_pretrained(model_name, torch_dtype=torch.float16, device_map="auto")+ tokenizer = AutoTokenizer.from_pretrained(model_name)+ autoround = AutoRound(model, tokenizer, bits=2, group_size=64)+ autoround.quantize()++ ##test auto_round format+ autoround.save_quantized(self.save_dir, format="auto_round", inplace=False)+ model_args = f"pretrained={self.save_dir}"+ res = simple_evaluate(model="hf", model_args=model_args,+ tasks=self.tasks,+ batch_size="auto")+ res = make_table(res) ##0.1985+ accuracy = get_accuracy(res)+ assert accuracy > 0.18+ shutil.rmtree("./saved", ignore_errors=True)+++ autoround.save_quantized(self.save_dir, format="auto_gptq", inplace=False)+ model_args = f"pretrained={self.save_dir}"+ res = simple_evaluate(model="hf", model_args=model_args,+ tasks=self.tasks,+ batch_size="auto")+ res = make_table(res) ##0.1985+ accuracy = get_accuracy(res)+ assert accuracy > 0.18+ shutil.rmtree("./saved", ignore_errors=True)++if __name__ == "__main__":+ unittest.main()\ No newline at end of filediff --git a/test/test_cuda_before_release.py b/test_cuda/test_main_func.pysimilarity index 68%rename from test/test_cuda_before_release.pyrename to test_cuda/test_main_func.pyindex 4e3c5df43..0b1c2b736 100644--- a/test/test_cuda_before_release.py+++ b/test_cuda/test_main_func.py@@ -9,7 +9,7 @@ import torchimport transformersfrom transformers import AutoModelForCausalLM, AutoTokenizer-from auto_round import AutoRound+from auto_round import AutoRound, AutoRoundAdamfrom auto_round.eval.evaluation import simple_evaluatefrom lm_eval.utils import make_table # pylint: disable=E0401@@ -24,7 +24,7 @@ def get_accuracy(data):return 0.0-class TestAutoRound(unittest.TestCase):+class TestMainFunc(unittest.TestCase):@classmethoddef setUpClass(self):self.save_dir = "./saved"@@ -35,7 +35,6 @@ class TestAutoRound(unittest.TestCase):shutil.rmtree("./saved", ignore_errors=True)shutil.rmtree("runs", ignore_errors=True)- @unittest.skipIf(torch.cuda.is_available() is False, "Skipping because no cuda")def test_backend(self):model_name = "/models/opt-125m"model = AutoModelForCausalLM.from_pretrained(model_name, torch_dtype=torch.float16, device_map="auto")@@ -54,7 +53,7 @@ class TestAutoRound(unittest.TestCase):assert accuracy > 0.35shutil.rmtree("./saved", ignore_errors=True)- ##test auto_round format+ ##test auto_gptq formatautoround.save_quantized(self.save_dir, format="auto_gptq", inplace=False)model_args = f"pretrained={self.save_dir}"res = simple_evaluate(model="hf", model_args=model_args,@@ -65,7 +64,7 @@ class TestAutoRound(unittest.TestCase):assert accuracy > 0.35shutil.rmtree("./saved", ignore_errors=True)- ##test auto_round format+ ##test auto_awq formatautoround.save_quantized(self.save_dir, format="auto_awq", inplace=False)model_args = f"pretrained={self.save_dir}"res = simple_evaluate(model="hf", model_args=model_args,@@ -113,27 +112,57 @@ class TestAutoRound(unittest.TestCase):@unittest.skipIf(torch.cuda.is_available() is False, "Skipping because no cuda")def test_undivided_group_size_tuning(self):- model_name = "/models/falcon-7b"+ model_name = "/models/opt-125m"model = AutoModelForCausalLM.from_pretrained(model_name, torch_dtype=torch.float16, device_map="auto")tokenizer = AutoTokenizer.from_pretrained(model_name)- autoround = AutoRound(model, tokenizer, bits=4, group_size=128, nsamples=1, iters=1)+ autoround = AutoRound(model, tokenizer, bits=4, group_size=127, nsamples=2, iters=2)+ autoround.quantize()+++ def test_adam(self):+ model_name = "/models/opt-125m"+ model = AutoModelForCausalLM.from_pretrained(model_name, torch_dtype=torch.float16, device_map="auto")+ tokenizer = AutoTokenizer.from_pretrained(model_name)+ autoround = AutoRoundAdam(model, tokenizer, bits=4, group_size=128)+ autoround.quantize()++ ##test auto_round format+ autoround.save_quantized(self.save_dir, format="auto_round", inplace=False)+ model_args = f"pretrained={self.save_dir}"+ res = simple_evaluate(model="hf", model_args=model_args,+ tasks=self.tasks,+ batch_size="auto")+ res = make_table(res)+ accuracy = get_accuracy(res)+ assert accuracy > 0.35+ shutil.rmtree("./saved", ignore_errors=True)++ def test_autoround_asym(self): ##need to install false+ try:+ from autoround_exllamav2_kernels import gemm_half_q_half, make_q_matrix+ except ImportError as e:+ print("skip autoround asym test, as autoround is not installed from source")+ return+ model_name = "/models/opt-125m"+ model = AutoModelForCausalLM.from_pretrained(model_name, torch_dtype=torch.float16, device_map="auto")+ tokenizer = AutoTokenizer.from_pretrained(model_name)+ autoround = AutoRound(model, tokenizer, bits=4, group_size=128, sym=False)autoround.quantize()++ ##test auto_round format+ autoround.save_quantized(self.save_dir, format="auto_round", inplace=False)+ model_args = f"pretrained={self.save_dir}"+ res = simple_evaluate(model="hf", model_args=model_args,+ tasks=self.tasks,+ batch_size="auto")+ res = make_table(res)+ accuracy = get_accuracy(res)+ assert accuracy > 0.35+ shutil.rmtree("./saved", ignore_errors=True)+++- @unittest.skipIf(torch.cuda.is_available() is False, "Skipping because no cuda")- def test_vision_generation(self):- quantized_model_path = "OPEA/Phi-3.5-vision-instruct-qvision-int4-sym-inc"- from auto_round import AutoRoundConfig- device = "auto" ##cpu, hpu, cuda- quantization_config = AutoRoundConfig(- backend=device- )- model = AutoModelForCausalLM.from_pretrained(quantized_model_path, trust_remote_code=True,- device_map=device, quantization_config=quantization_config)- tokenizer = AutoTokenizer.from_pretrained(quantized_model_path)- text = "There is a girl who likes adventure,"- inputs = tokenizer(text, return_tensors="pt").to(model.device)- res = tokenizer.decode(model.generate(**inputs, max_new_tokens=50)[0])- print(res)- assert (- res == """<s> There is a girl who likes adventure, and she is looking for a partner to go on a treasure hunt. She has found a map that leads to a hidden treasure, but she needs a partner to help her decipher the clues and find the treasure. You""")+if __name__ == "__main__":+ unittest.main()\ No newline at end of filediff --git a/test_cuda/test_multiple_card_calib.py b/test_cuda/test_multiple_card_calib.pynew file mode 100644index 000000000..a56fe1b29--- /dev/null+++ b/test_cuda/test_multiple_card_calib.py@@ -0,0 +1,49 @@+import copy+import shutil+import sys+import unittest+import re++sys.path.insert(0, "..")+import torch+import transformers+from transformers import AutoModelForCausalLM, AutoTokenizer++from auto_round import AutoRound+from auto_round.eval.evaluation import simple_evaluate+from lm_eval.utils import make_table # pylint: disable=E0401+import os++def get_accuracy(data):+ match = re.search(r'\|acc\s+\|[↑↓]\s+\|\s+([\d.]+)\|', data)++ if match:+ accuracy = float(match.group(1))+ return accuracy+ else:+ return 0.0+++class TestAutoRound(unittest.TestCase):+ @classmethod+ def setUpClass(self):+ self.save_dir = "./saved"+ self.tasks = "lambada_openai"++ @classmethod+ def tearDownClass(self):+ shutil.rmtree("./saved", ignore_errors=True)+ shutil.rmtree("runs", ignore_errors=True)++ def test_multiple_card_calib(self):+ python_path = sys.executable++ ##test llm script+ res = os.system(+ f"cd .. && {python_path} -m auto_round --model /models/Meta-Llama-3.1-8B-Instruct --devices '0,1' --quant_lm_head --disable_eval --iters 1 --nsamples 1 --output_dir None")+++if __name__ == "__main__":+ unittest.main()++diff --git a/test_cuda/test_support_vlms.py b/test_cuda/test_support_vlms.pynew file mode 100644index 000000000..5aae0572b--- /dev/null+++ b/test_cuda/test_support_vlms.py@@ -0,0 +1,262 @@+import os+import sys+import shutil+import unittest++sys.path.insert(0, '..')++from auto_round import AutoRoundConfig ## must import for auto-round format+import requests+from PIL import Image+++class TestSupportVLMS(unittest.TestCase):+ @classmethod+ def setUpClass(self):+ self.save_dir = os.path.join(os.path.dirname(__file__), "./ut_saved")+ self.python_path = sys.executable+ self.device = 0++ @classmethod+ def tearDownClass(self):+ shutil.rmtree(self.save_dir, ignore_errors=True)++ def test_qwen2(self):+ model_path = "/models/Qwen2-VL-2B-Instruct/"+ # test tune+ res = os.system(+ f"cd .. && {self.python_path} -m auto_round --mllm "+ f"--model {model_path} --iter 2 --output_dir {self.save_dir} --device {self.device}")+ self.assertFalse(res > 0 or res == -1, msg="qwen2 tuning fail")++ # test infer+ quantized_model_path = os.path.join(self.save_dir, "Qwen2-VL-2B-Instruct-w4g128-auto_round")++ from transformers import Qwen2VLForConditionalGeneration, AutoProcessor+ model = Qwen2VLForConditionalGeneration.from_pretrained(+ quantized_model_path,+ torch_dtype="auto",+ device_map=f"cuda:{self.device}",+ )+ processor = AutoProcessor.from_pretrained(quantized_model_path)+ image_url = "https://qianwen-res.oss-cn-beijing.aliyuncs.com/Qwen-VL/assets/demo.jpeg"+ messages = [+ {+ "role": "user",+ "content": [+ {+ "type": "image",+ "image": image_url,+ },+ {"type": "text", "text": "Describe this image."},+ ],+ }+ ]++ # Preparation for inference+ text = processor.apply_chat_template(+ messages, tokenize=False, add_generation_prompt=True+ )+ image_inputs = Image.open(requests.get(image_url, stream=True).raw)+ inputs = processor(+ text=[text],+ images=image_inputs,+ padding=True,+ return_tensors="pt",+ )+ inputs = inputs.to(model.device)++ generated_ids = model.generate(**inputs, max_new_tokens=128)+ generated_ids_trimmed = [+ out_ids[len(in_ids) :] for in_ids, out_ids in zip(inputs.input_ids, generated_ids)+ ]+ output_text = processor.batch_decode(+ generated_ids_trimmed, skip_special_tokens=True, clean_up_tokenization_spaces=False+ )+ print(output_text[0])++ def test_phi3(self):+ model_path = "/models/Phi-3.5-vision-instruct/"+ ## test tune+ res = os.system(+ f"cd .. && {self.python_path} -m auto_round --mllm "+ f"--model {model_path} --iter 2 --output_dir {self.save_dir} --device {self.device}")+ self.assertFalse(res > 0 or res == -1, msg="Phi-3.5 tuning fail")++ ## test infer+ from transformers import AutoModelForCausalLM, AutoProcessor+ quantized_model_path = os.path.join(self.save_dir, "Phi-3.5-vision-instruct-w4g128-auto_round")+ res = os.system(f"cp /models/Phi-3.5-vision-instruct/*.py {quantized_model_path}")+ model = AutoModelForCausalLM.from_pretrained(+ quantized_model_path,+ device_map=f"cuda:{self.device}",+ trust_remote_code=True,+ torch_dtype="auto"+ )+ processor = AutoProcessor.from_pretrained(quantized_model_path,+ trust_remote_code=True,+ num_crops=4+ )++ image_url = "https://qianwen-res.oss-cn-beijing.aliyuncs.com/Qwen-VL/assets/demo.jpeg"+ content = "Describe this image."+ messages = [+ {"role": "user",+ "content": "<|image_1|>\n"+content},+ ]++ prompt = processor.tokenizer.apply_chat_template(+ messages,+ tokenize=False,+ add_generation_prompt=True+ )+ image_inputs = Image.open(requests.get(image_url, stream=True).raw)+ inputs = processor(prompt, image_inputs, return_tensors="pt").to(model.device)++ generation_args = {+ "max_new_tokens": 1000,+ "temperature": 0.0,+ "do_sample": False,+ }++ generate_ids = model.generate(**inputs,+ eos_token_id=processor.tokenizer.eos_token_id,+ **generation_args+ )++ # remove input tokens+ generate_ids = generate_ids[:, inputs['input_ids'].shape[1]:]+ response = processor.batch_decode(generate_ids,+ skip_special_tokens=True,+ clean_up_tokenization_spaces=False)[0]++ print(response)++ def test_llava(self):+ model_path = "/models/llava-v1.5-7b/"+ ## test tune+ res = os.system(+ f"cd .. && {self.python_path} -m auto_round --mllm "+ f"--model {model_path} --iter 2 --output_dir {self.save_dir} --device {self.device}")+ self.assertFalse(res > 0 or res == -1, msg="llava-v1.5-7b tuning fail")++ ## test infer+ from llava.model.builder import load_pretrained_model+ from llava.train.train import preprocess, preprocess_multimodal+ class DataArgs:+ is_multimodal = True+ mm_use_im_start_end = False++ quantized_model_path = os.path.join(self.save_dir, "llava-v1.5-7b-w4g128-auto_round")+ tokenizer, model, image_processor, _ = load_pretrained_model(+ quantized_model_path,+ model_base=None,+ model_name=quantized_model_path,+ torch_dtype="auto",+ device_map=f"cuda:{self.device}",+ )+ image_url = "http://images.cocodataset.org/train2017/000000116003.jpg"+ messages = [{"from": "human", "value": "What is the tennis player doing in the image?\n<image>"}]++ # Preparation for inference+ image = Image.open(requests.get(image_url, stream=True).raw).convert('RGB')+ image_input = image_processor.preprocess(image, return_tensors='pt')['pixel_values'][0].to(model.device)+ input_data = preprocess_multimodal([messages], DataArgs())+ inputs = preprocess(input_data, tokenizer, has_image=(image_input is not None))++ output = model.generate(inputs['input_ids'].to(model.device), images=image_input.unsqueeze(0).half(), max_new_tokens=50)+ print(tokenizer.batch_decode(output))++ def test_llama(self):+ model_path = "/models/Llama-3.2-11B-Vision-Instruct/"+ ## test tune+ res = os.system(+ f"cd .. && {self.python_path} -m auto_round --mllm "+ f"--model {model_path} --iter 2 --output_dir {self.save_dir} --device {self.device}")+ self.assertFalse(res > 0 or res == -1, msg="llama-3.2 tuning fail")++ ## test infer+ from transformers import MllamaForConditionalGeneration, AutoProcessor+ quantized_model_path = os.path.join(self.save_dir, "Llama-3.2-11B-Vision-Instruct-w4g128-auto_round")+ model = MllamaForConditionalGeneration.from_pretrained(+ quantized_model_path,+ torch_dtype="auto",+ device_map=f"cuda:{self.device}",+ )+ processor = AutoProcessor.from_pretrained(quantized_model_path)+ image_url = "https://huggingface.co/datasets/huggingface/documentation-images/resolve/0052a70beed5bf71b92610a43a52df6d286cd5f3/diffusers/rabbit.jpg"+ messages = [+ {"role": "user", "content": [+ {"type": "image"},+ {"type": "text", "text": "Please write a haiku for this one, it would be: "}+ ]}+ ]++ # Preparation for inference+ image = Image.open(requests.get(image_url, stream=True).raw)+ input_text = processor.apply_chat_template(messages, add_generation_prompt=True)+ inputs = processor(+ image,+ input_text,+ add_special_tokens=False,+ return_tensors="pt"+ ).to(model.device)++ output = model.generate(**inputs, max_new_tokens=50)+ print(processor.decode(output[0]))++ def test_cogvlm(self):+ model_path = "/models/cogvlm2-llama3-chat-19B/"+ ## test tune+ res = os.system(+ f"cd .. && {self.python_path} -m auto_round --mllm "+ f"--model {model_path} --iter 2 --output_dir {self.save_dir} --device {self.device}")+ self.assertFalse(res > 0 or res == -1, msg="cogvlm2 tuning fail")++ ## test infer+ DEVICE = f"cuda:{self.device}"+ from transformers import AutoModelForCausalLM, AutoTokenizer+ quantized_model_path = os.path.join(self.save_dir, "cogvlm2-llama3-chat-19B-w4g128-auto_round")+ tokenizer = AutoTokenizer.from_pretrained(+ quantized_model_path,+ trust_remote_code=True+ )+ model = AutoModelForCausalLM.from_pretrained(+ quantized_model_path,+ torch_dtype="auto",+ trust_remote_code=True,+ device_map=DEVICE,+ ).to(DEVICE).eval()++ image_url = "https://qianwen-res.oss-cn-beijing.aliyuncs.com/Qwen-VL/assets/demo.jpeg"+ content = "Describe this image."++ text_only_template = "A chat between a curious user and an artificial intelligence assistant. The assistant gives helpful, detailed, and polite answers to the user's questions. USER: {} ASSISTANT:"+ query = text_only_template.format(content)++ image = Image.open(requests.get(image_url, stream=True).raw)+ input_by_model = model.build_conversation_input_ids(+ tokenizer,+ query=query,+ images=[image],+ template_version='chat'+ )+ inputs = {+ 'input_ids': input_by_model['input_ids'].unsqueeze(0).to(DEVICE),+ 'token_type_ids': input_by_model['token_type_ids'].unsqueeze(0).to(DEVICE),+ 'attention_mask': input_by_model['attention_mask'].unsqueeze(0).to(DEVICE),+ 'images': [[input_by_model['images'][0].to(DEVICE).to(model.dtype)]] if image is not None else None,+ }+ gen_kwargs = {+ "max_new_tokens": 2048,+ "pad_token_id": 128002,+ }++ outputs = model.generate(**inputs, **gen_kwargs)+ outputs = outputs[:, inputs['input_ids'].shape[1]:]+ response = tokenizer.decode(outputs[0])+ response = response.split("<|end_of_text|>")[0]+ print(response)++if __name__ == "__main__":+ unittest.main()\ No newline at end of filediff --git a/test_cuda/test_vlms.py b/test_cuda/test_vlms.pynew file mode 100644index 000000000..ad112cbce--- /dev/null+++ b/test_cuda/test_vlms.py@@ -0,0 +1,147 @@+import copy+import shutil+import sys+import unittest+import re++sys.path.insert(0, "..")++from PIL import Image+import requests+++class TestAutoRound(unittest.TestCase):+ @classmethod+ def setUpClass(self):+ self.save_dir = "./saved"++ @classmethod+ def tearDownClass(self):+ shutil.rmtree(self.save_dir, ignore_errors=True)+ shutil.rmtree("runs", ignore_errors=True)++ #+ # def test_vision_generation(self):+ # quantized_model_path = "OPEA/Phi-3.5-vision-instruct-qvision-int4-sym-inc"+ # from auto_round import AutoRoundConfig+ # device = "auto" ##cpu, hpu, cuda+ # quantization_config = AutoRoundConfig(+ # backend=device+ # )+ # model = AutoModelForCausalLM.from_pretrained(quantized_model_path, trust_remote_code=True,+ # device_map=device, quantization_config=quantization_config)+ # tokenizer = AutoTokenizer.from_pretrained(quantized_model_path)+ # text = "There is a girl who likes adventure,"+ # inputs = tokenizer(text, return_tensors="pt").to(model.device)+ # res = tokenizer.decode(model.generate(**inputs, max_new_tokens=50)[0])+ # print(res)+ # assert (+ # res == """<s> There is a girl who likes adventure, and she is looking for a partner to go on a treasure hunt. She has found a map that leads to a hidden treasure, but she needs a partner to help her decipher the clues and find the treasure. You""")++ def inference(self, quantized_model_dir):+ from transformers import Qwen2VLForConditionalGeneration, AutoProcessor, AutoTokenizer+ tokenizer = AutoTokenizer.from_pretrained(quantized_model_dir)+ processor = AutoProcessor.from_pretrained(quantized_model_dir, trust_remote_code=True)+ model = Qwen2VLForConditionalGeneration.from_pretrained(+ quantized_model_dir,+ torch_dtype="auto",+ device_map="auto",+ ##revision="df7f44c" ##AutoGPTQ format+ )++ image_url = "../docs/imgs/norm_bias_overview.png"+ messages = [+ {+ "role": "user",+ "content": [+ {+ "type": "image",+ "image": image_url,+ },+ {"type": "text", "text": "Describe this image."},+ ],+ }+ ]++ # Preparation for inference+ text = processor.apply_chat_template(+ messages, tokenize=False, add_generation_prompt=True+ )+ image_inputs = Image.open(image_url)+ inputs = processor(+ text=[text],+ images=image_inputs,+ padding=True,+ return_tensors="pt",+ )+ inputs = inputs.to(model.device)++ generated_ids = model.generate(**inputs, max_new_tokens=128)+ generated_ids_trimmed = [+ out_ids[len(in_ids):] for in_ids, out_ids in zip(inputs.input_ids, generated_ids)+ ]+ output_text = processor.batch_decode(+ generated_ids_trimmed, skip_special_tokens=True, clean_up_tokenization_spaces=False+ )+ print(output_text[0])++ def test_vlm_tune(self):+ from auto_round import AutoRoundMLLM+ from transformers import Qwen2VLForConditionalGeneration, AutoProcessor, AutoTokenizer++ ## load the model+ model_name = "/models/Qwen2-VL-2B-Instruct"+ model = Qwen2VLForConditionalGeneration.from_pretrained(+ model_name, trust_remote_code=True)+ tokenizer = AutoTokenizer.from_pretrained(model_name)+ processor = AutoProcessor.from_pretrained(model_name, trust_remote_code=True)++ ## quantize the model+ bits, group_size, sym = 4, 128, True+ autoround = AutoRoundMLLM(model, tokenizer, processor,+ bits=bits, group_size=group_size, sym=sym, iters=1, nsamples=1)+ autoround.quantize()++ quantized_model_path = self.save_dir+ autoround.save_quantized(quantized_model_path, format='auto_round', inplace=False)+ self.inference(quantized_model_path)+ shutil.rmtree(self.save_dir, ignore_errors=True)+ autoround.save_quantized(quantized_model_path, format='auto_gptq', inplace=False)+ self.inference(quantized_model_path)+ shutil.rmtree(self.save_dir, ignore_errors=True)++ def test_quant_not_text(self):+ from auto_round import AutoRoundMLLM+ from transformers import Qwen2VLForConditionalGeneration, AutoProcessor, AutoTokenizer++ ## load the model+ model_name = "/models/Qwen2-VL-2B-Instruct"+ model = Qwen2VLForConditionalGeneration.from_pretrained(+ model_name, trust_remote_code=True)+ tokenizer = AutoTokenizer.from_pretrained(model_name)+ processor = AutoProcessor.from_pretrained(model_name, trust_remote_code=True)++ ## quantize the model+ bits, group_size, sym = 4, 128, True+ autoround = AutoRoundMLLM(model, tokenizer, processor,+ bits=bits, group_size=group_size, sym=sym, iters=1, nsamples=1,quant_nontext_module=True)+ autoround.quantize()++ quantized_model_path = "./saved"+ autoround.save_quantized(quantized_model_path, format='auto_round', inplace=False)+ self.inference(quantized_model_path)+ shutil.rmtree("./saved", ignore_errors=True)++ def test_quant_not_text_fp_layers(self):+ import os+ python_path = sys.executable+ absolute_path = os.path.abspath(self.save_dir)+ res = os.system(+ f"cd .. && {python_path} -m auto_round --mllm --model /models/Qwen2-VL-2B-Instruct --fp_layers model.layers.27,visual.blocks.29 --quant_nontext_module --iters 1 --nsamples 1 --output_dir {absolute_path}")+ self.inference(os.path.join(absolute_path,"Qwen2-VL-2B-Instruct-w4g128-auto_round"))+ shutil.rmtree(os.path.join(absolute_path,"Qwen2-VL-2B-Instruct-w4g128-auto_round"), ignore_errors=True)++++if __name__ == "__main__":+ unittest.main()