Skip to content

PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度

Add gpu ut (#370)

Wenhua Chengecc17be01b00
diff --git a/test_cuda/test_2_3bits.py b/test_cuda/test_2_3bits.py
new file mode 100644
index 000000000..361b87e67
--- /dev/null
+++ b/test_cuda/test_2_3bits.py
@@ -0,0 +1,104 @@
+import copy
+import shutil
+import sys
+import unittest
+import re
+
+sys.path.insert(0, "..")
+import torch
+import transformers
+from transformers import AutoModelForCausalLM, AutoTokenizer
+
+from auto_round import AutoRound
+from auto_round.eval.evaluation import simple_evaluate
+from lm_eval.utils import make_table # pylint: disable=E0401
+
+
+def get_accuracy(data):
+ match = re.search(r'\|acc\s+\|[↑↓]\s+\|\s+([\d.]+)\|', data)
+
+ if match:
+ accuracy = float(match.group(1))
+ return accuracy
+ else:
+ return 0.0
+
+
+class TestAutoRound(unittest.TestCase):
+ @classmethod
+ def setUpClass(self):
+ self.save_dir = "./saved"
+ self.tasks = "lambada_openai"
+
+ @classmethod
+ def tearDownClass(self):
+ shutil.rmtree("./saved", ignore_errors=True)
+ shutil.rmtree("runs", ignore_errors=True)
+
+ def test_3bits_autogptq(self):
+ model_name = "/models/opt-125m"
+ model = AutoModelForCausalLM.from_pretrained(model_name, torch_dtype=torch.float16, device_map="auto")
+ tokenizer = AutoTokenizer.from_pretrained(model_name)
+ autoround = AutoRound(model, tokenizer, bits=3)
+ autoround.quantize()
+
+ autoround.save_quantized(self.save_dir, format="auto_gptq", inplace=False)
+ model_args = f"pretrained={self.save_dir}"
+ res = simple_evaluate(model="hf", model_args=model_args,
+ tasks=self.tasks,
+ batch_size="auto")
+ res = make_table(res)
+
+ accuracy = get_accuracy(res)
+ assert accuracy > 0.30
+ shutil.rmtree("./saved", ignore_errors=True)
+
+ def test_norm_bias_tuning(self):
+ model_name = "/models/opt-125m"
+ model = AutoModelForCausalLM.from_pretrained(model_name, torch_dtype=torch.float16, device_map="auto")
+ tokenizer = AutoTokenizer.from_pretrained(model_name)
+ autoround = AutoRound(model, tokenizer, bits=2, group_size=64, enable_norm_bias_tuning=True)
+ autoround.quantize()
+
+ ##test auto_round format
+ autoround.save_quantized(self.save_dir, format="auto_round", inplace=False)
+ model_args = f"pretrained={self.save_dir}"
+ res = simple_evaluate(model="hf", model_args=model_args,
+ tasks=self.tasks,
+ batch_size="auto")
+ res = make_table(res) ##0.2212
+ accuracy = get_accuracy(res)
+ assert accuracy > 0.20
+ shutil.rmtree("./saved", ignore_errors=True)
+
+ def test_2bits_autoround(self):
+ model_name = "/models/opt-125m"
+ model = AutoModelForCausalLM.from_pretrained(model_name, torch_dtype=torch.float16, device_map="auto")
+ tokenizer = AutoTokenizer.from_pretrained(model_name)
+ autoround = AutoRound(model, tokenizer, bits=2, group_size=64)
+ autoround.quantize()
+
+ ##test auto_round format
+ autoround.save_quantized(self.save_dir, format="auto_round", inplace=False)
+ model_args = f"pretrained={self.save_dir}"
+ res = simple_evaluate(model="hf", model_args=model_args,
+ tasks=self.tasks,
+ batch_size="auto")
+ res = make_table(res) ##0.1985
+ accuracy = get_accuracy(res)
+ assert accuracy > 0.18
+ shutil.rmtree("./saved", ignore_errors=True)
+
+
+ autoround.save_quantized(self.save_dir, format="auto_gptq", inplace=False)
+ model_args = f"pretrained={self.save_dir}"
+ res = simple_evaluate(model="hf", model_args=model_args,
+ tasks=self.tasks,
+ batch_size="auto")
+ res = make_table(res) ##0.1985
+ accuracy = get_accuracy(res)
+ assert accuracy > 0.18
+ shutil.rmtree("./saved", ignore_errors=True)
+
+if __name__ == "__main__":
+ unittest.main()
\ No newline at end of file
diff --git a/test/test_cuda_before_release.py b/test_cuda/test_main_func.py
similarity index 68%
rename from test/test_cuda_before_release.py
rename to test_cuda/test_main_func.py
index 4e3c5df43..0b1c2b736 100644
--- a/test/test_cuda_before_release.py
+++ b/test_cuda/test_main_func.py
@@ -9,7 +9,7 @@ import torch
import transformers
from transformers import AutoModelForCausalLM, AutoTokenizer
-from auto_round import AutoRound
+from auto_round import AutoRound, AutoRoundAdam
from auto_round.eval.evaluation import simple_evaluate
from lm_eval.utils import make_table # pylint: disable=E0401
@@ -24,7 +24,7 @@ def get_accuracy(data):
return 0.0
-class TestAutoRound(unittest.TestCase):
+class TestMainFunc(unittest.TestCase):
@classmethod
def setUpClass(self):
self.save_dir = "./saved"
@@ -35,7 +35,6 @@ class TestAutoRound(unittest.TestCase):
shutil.rmtree("./saved", ignore_errors=True)
shutil.rmtree("runs", ignore_errors=True)
- @unittest.skipIf(torch.cuda.is_available() is False, "Skipping because no cuda")
def test_backend(self):
model_name = "/models/opt-125m"
model = AutoModelForCausalLM.from_pretrained(model_name, torch_dtype=torch.float16, device_map="auto")
@@ -54,7 +53,7 @@ class TestAutoRound(unittest.TestCase):
assert accuracy > 0.35
shutil.rmtree("./saved", ignore_errors=True)
- ##test auto_round format
+ ##test auto_gptq format
autoround.save_quantized(self.save_dir, format="auto_gptq", inplace=False)
model_args = f"pretrained={self.save_dir}"
res = simple_evaluate(model="hf", model_args=model_args,
@@ -65,7 +64,7 @@ class TestAutoRound(unittest.TestCase):
assert accuracy > 0.35
shutil.rmtree("./saved", ignore_errors=True)
- ##test auto_round format
+ ##test auto_awq format
autoround.save_quantized(self.save_dir, format="auto_awq", inplace=False)
model_args = f"pretrained={self.save_dir}"
res = simple_evaluate(model="hf", model_args=model_args,
@@ -113,27 +112,57 @@ class TestAutoRound(unittest.TestCase):
@unittest.skipIf(torch.cuda.is_available() is False, "Skipping because no cuda")
def test_undivided_group_size_tuning(self):
- model_name = "/models/falcon-7b"
+ model_name = "/models/opt-125m"
model = AutoModelForCausalLM.from_pretrained(model_name, torch_dtype=torch.float16, device_map="auto")
tokenizer = AutoTokenizer.from_pretrained(model_name)
- autoround = AutoRound(model, tokenizer, bits=4, group_size=128, nsamples=1, iters=1)
+ autoround = AutoRound(model, tokenizer, bits=4, group_size=127, nsamples=2, iters=2)
+ autoround.quantize()
+
+
+ def test_adam(self):
+ model_name = "/models/opt-125m"
+ model = AutoModelForCausalLM.from_pretrained(model_name, torch_dtype=torch.float16, device_map="auto")
+ tokenizer = AutoTokenizer.from_pretrained(model_name)
+ autoround = AutoRoundAdam(model, tokenizer, bits=4, group_size=128)
+ autoround.quantize()
+
+ ##test auto_round format
+ autoround.save_quantized(self.save_dir, format="auto_round", inplace=False)
+ model_args = f"pretrained={self.save_dir}"
+ res = simple_evaluate(model="hf", model_args=model_args,
+ tasks=self.tasks,
+ batch_size="auto")
+ res = make_table(res)
+ accuracy = get_accuracy(res)
+ assert accuracy > 0.35
+ shutil.rmtree("./saved", ignore_errors=True)
+
+ def test_autoround_asym(self): ##need to install false
+ try:
+ from autoround_exllamav2_kernels import gemm_half_q_half, make_q_matrix
+ except ImportError as e:
+ print("skip autoround asym test, as autoround is not installed from source")
+ return
+ model_name = "/models/opt-125m"
+ model = AutoModelForCausalLM.from_pretrained(model_name, torch_dtype=torch.float16, device_map="auto")
+ tokenizer = AutoTokenizer.from_pretrained(model_name)
+ autoround = AutoRound(model, tokenizer, bits=4, group_size=128, sym=False)
autoround.quantize()
+
+ ##test auto_round format
+ autoround.save_quantized(self.save_dir, format="auto_round", inplace=False)
+ model_args = f"pretrained={self.save_dir}"
+ res = simple_evaluate(model="hf", model_args=model_args,
+ tasks=self.tasks,
+ batch_size="auto")
+ res = make_table(res)
+ accuracy = get_accuracy(res)
+ assert accuracy > 0.35
+ shutil.rmtree("./saved", ignore_errors=True)
+
+
+
- @unittest.skipIf(torch.cuda.is_available() is False, "Skipping because no cuda")
- def test_vision_generation(self):
- quantized_model_path = "OPEA/Phi-3.5-vision-instruct-qvision-int4-sym-inc"
- from auto_round import AutoRoundConfig
- device = "auto" ##cpu, hpu, cuda
- quantization_config = AutoRoundConfig(
- backend=device
- )
- model = AutoModelForCausalLM.from_pretrained(quantized_model_path, trust_remote_code=True,
- device_map=device, quantization_config=quantization_config)
- tokenizer = AutoTokenizer.from_pretrained(quantized_model_path)
- text = "There is a girl who likes adventure,"
- inputs = tokenizer(text, return_tensors="pt").to(model.device)
- res = tokenizer.decode(model.generate(**inputs, max_new_tokens=50)[0])
- print(res)
- assert (
- res == """<s> There is a girl who likes adventure, and she is looking for a partner to go on a treasure hunt. She has found a map that leads to a hidden treasure, but she needs a partner to help her decipher the clues and find the treasure. You""")
+if __name__ == "__main__":
+ unittest.main()
\ No newline at end of file
diff --git a/test_cuda/test_multiple_card_calib.py b/test_cuda/test_multiple_card_calib.py
new file mode 100644
index 000000000..a56fe1b29
--- /dev/null
+++ b/test_cuda/test_multiple_card_calib.py
@@ -0,0 +1,49 @@
+import copy
+import shutil
+import sys
+import unittest
+import re
+
+sys.path.insert(0, "..")
+import torch
+import transformers
+from transformers import AutoModelForCausalLM, AutoTokenizer
+
+from auto_round import AutoRound
+from auto_round.eval.evaluation import simple_evaluate
+from lm_eval.utils import make_table # pylint: disable=E0401
+import os
+
+def get_accuracy(data):
+ match = re.search(r'\|acc\s+\|[↑↓]\s+\|\s+([\d.]+)\|', data)
+
+ if match:
+ accuracy = float(match.group(1))
+ return accuracy
+ else:
+ return 0.0
+
+
+class TestAutoRound(unittest.TestCase):
+ @classmethod
+ def setUpClass(self):
+ self.save_dir = "./saved"
+ self.tasks = "lambada_openai"
+
+ @classmethod
+ def tearDownClass(self):
+ shutil.rmtree("./saved", ignore_errors=True)
+ shutil.rmtree("runs", ignore_errors=True)
+
+ def test_multiple_card_calib(self):
+ python_path = sys.executable
+
+ ##test llm script
+ res = os.system(
+ f"cd .. && {python_path} -m auto_round --model /models/Meta-Llama-3.1-8B-Instruct --devices '0,1' --quant_lm_head --disable_eval --iters 1 --nsamples 1 --output_dir None")
+
+
+if __name__ == "__main__":
+ unittest.main()
+
+
diff --git a/test_cuda/test_support_vlms.py b/test_cuda/test_support_vlms.py
new file mode 100644
index 000000000..5aae0572b
--- /dev/null
+++ b/test_cuda/test_support_vlms.py
@@ -0,0 +1,262 @@
+import os
+import sys
+import shutil
+import unittest
+
+sys.path.insert(0, '..')
+
+from auto_round import AutoRoundConfig ## must import for auto-round format
+import requests
+from PIL import Image
+
+
+class TestSupportVLMS(unittest.TestCase):
+ @classmethod
+ def setUpClass(self):
+ self.save_dir = os.path.join(os.path.dirname(__file__), "./ut_saved")
+ self.python_path = sys.executable
+ self.device = 0
+
+ @classmethod
+ def tearDownClass(self):
+ shutil.rmtree(self.save_dir, ignore_errors=True)
+
+ def test_qwen2(self):
+ model_path = "/models/Qwen2-VL-2B-Instruct/"
+ # test tune
+ res = os.system(
+ f"cd .. && {self.python_path} -m auto_round --mllm "
+ f"--model {model_path} --iter 2 --output_dir {self.save_dir} --device {self.device}")
+ self.assertFalse(res > 0 or res == -1, msg="qwen2 tuning fail")
+
+ # test infer
+ quantized_model_path = os.path.join(self.save_dir, "Qwen2-VL-2B-Instruct-w4g128-auto_round")
+
+ from transformers import Qwen2VLForConditionalGeneration, AutoProcessor
+ model = Qwen2VLForConditionalGeneration.from_pretrained(
+ quantized_model_path,
+ torch_dtype="auto",
+ device_map=f"cuda:{self.device}",
+ )
+ processor = AutoProcessor.from_pretrained(quantized_model_path)
+ image_url = "https://qianwen-res.oss-cn-beijing.aliyuncs.com/Qwen-VL/assets/demo.jpeg"
+ messages = [
+ {
+ "role": "user",
+ "content": [
+ {
+ "type": "image",
+ "image": image_url,
+ },
+ {"type": "text", "text": "Describe this image."},
+ ],
+ }
+ ]
+
+ # Preparation for inference
+ text = processor.apply_chat_template(
+ messages, tokenize=False, add_generation_prompt=True
+ )
+ image_inputs = Image.open(requests.get(image_url, stream=True).raw)
+ inputs = processor(
+ text=[text],
+ images=image_inputs,
+ padding=True,
+ return_tensors="pt",
+ )
+ inputs = inputs.to(model.device)
+
+ generated_ids = model.generate(**inputs, max_new_tokens=128)
+ generated_ids_trimmed = [
+ out_ids[len(in_ids) :] for in_ids, out_ids in zip(inputs.input_ids, generated_ids)
+ ]
+ output_text = processor.batch_decode(
+ generated_ids_trimmed, skip_special_tokens=True, clean_up_tokenization_spaces=False
+ )
+ print(output_text[0])
+
+ def test_phi3(self):
+ model_path = "/models/Phi-3.5-vision-instruct/"
+ ## test tune
+ res = os.system(
+ f"cd .. && {self.python_path} -m auto_round --mllm "
+ f"--model {model_path} --iter 2 --output_dir {self.save_dir} --device {self.device}")
+ self.assertFalse(res > 0 or res == -1, msg="Phi-3.5 tuning fail")
+
+ ## test infer
+ from transformers import AutoModelForCausalLM, AutoProcessor
+ quantized_model_path = os.path.join(self.save_dir, "Phi-3.5-vision-instruct-w4g128-auto_round")
+ res = os.system(f"cp /models/Phi-3.5-vision-instruct/*.py {quantized_model_path}")
+ model = AutoModelForCausalLM.from_pretrained(
+ quantized_model_path,
+ device_map=f"cuda:{self.device}",
+ trust_remote_code=True,
+ torch_dtype="auto"
+ )
+ processor = AutoProcessor.from_pretrained(quantized_model_path,
+ trust_remote_code=True,
+ num_crops=4
+ )
+
+ image_url = "https://qianwen-res.oss-cn-beijing.aliyuncs.com/Qwen-VL/assets/demo.jpeg"
+ content = "Describe this image."
+ messages = [
+ {"role": "user",
+ "content": "<|image_1|>\n"+content},
+ ]
+
+ prompt = processor.tokenizer.apply_chat_template(
+ messages,
+ tokenize=False,
+ add_generation_prompt=True
+ )
+ image_inputs = Image.open(requests.get(image_url, stream=True).raw)
+ inputs = processor(prompt, image_inputs, return_tensors="pt").to(model.device)
+
+ generation_args = {
+ "max_new_tokens": 1000,
+ "temperature": 0.0,
+ "do_sample": False,
+ }
+
+ generate_ids = model.generate(**inputs,
+ eos_token_id=processor.tokenizer.eos_token_id,
+ **generation_args
+ )
+
+ # remove input tokens
+ generate_ids = generate_ids[:, inputs['input_ids'].shape[1]:]
+ response = processor.batch_decode(generate_ids,
+ skip_special_tokens=True,
+ clean_up_tokenization_spaces=False)[0]
+
+ print(response)
+
+ def test_llava(self):
+ model_path = "/models/llava-v1.5-7b/"
+ ## test tune
+ res = os.system(
+ f"cd .. && {self.python_path} -m auto_round --mllm "
+ f"--model {model_path} --iter 2 --output_dir {self.save_dir} --device {self.device}")
+ self.assertFalse(res > 0 or res == -1, msg="llava-v1.5-7b tuning fail")
+
+ ## test infer
+ from llava.model.builder import load_pretrained_model
+ from llava.train.train import preprocess, preprocess_multimodal
+ class DataArgs:
+ is_multimodal = True
+ mm_use_im_start_end = False
+
+ quantized_model_path = os.path.join(self.save_dir, "llava-v1.5-7b-w4g128-auto_round")
+ tokenizer, model, image_processor, _ = load_pretrained_model(
+ quantized_model_path,
+ model_base=None,
+ model_name=quantized_model_path,
+ torch_dtype="auto",
+ device_map=f"cuda:{self.device}",
+ )
+ image_url = "http://images.cocodataset.org/train2017/000000116003.jpg"
+ messages = [{"from": "human", "value": "What is the tennis player doing in the image?\n<image>"}]
+
+ # Preparation for inference
+ image = Image.open(requests.get(image_url, stream=True).raw).convert('RGB')
+ image_input = image_processor.preprocess(image, return_tensors='pt')['pixel_values'][0].to(model.device)
+ input_data = preprocess_multimodal([messages], DataArgs())
+ inputs = preprocess(input_data, tokenizer, has_image=(image_input is not None))
+
+ output = model.generate(inputs['input_ids'].to(model.device), images=image_input.unsqueeze(0).half(), max_new_tokens=50)
+ print(tokenizer.batch_decode(output))
+
+ def test_llama(self):
+ model_path = "/models/Llama-3.2-11B-Vision-Instruct/"
+ ## test tune
+ res = os.system(
+ f"cd .. && {self.python_path} -m auto_round --mllm "
+ f"--model {model_path} --iter 2 --output_dir {self.save_dir} --device {self.device}")
+ self.assertFalse(res > 0 or res == -1, msg="llama-3.2 tuning fail")
+
+ ## test infer
+ from transformers import MllamaForConditionalGeneration, AutoProcessor
+ quantized_model_path = os.path.join(self.save_dir, "Llama-3.2-11B-Vision-Instruct-w4g128-auto_round")
+ model = MllamaForConditionalGeneration.from_pretrained(
+ quantized_model_path,
+ torch_dtype="auto",
+ device_map=f"cuda:{self.device}",
+ )
+ processor = AutoProcessor.from_pretrained(quantized_model_path)
+ image_url = "https://huggingface.co/datasets/huggingface/documentation-images/resolve/0052a70beed5bf71b92610a43a52df6d286cd5f3/diffusers/rabbit.jpg"
+ messages = [
+ {"role": "user", "content": [
+ {"type": "image"},
+ {"type": "text", "text": "Please write a haiku for this one, it would be: "}
+ ]}
+ ]
+
+ # Preparation for inference
+ image = Image.open(requests.get(image_url, stream=True).raw)
+ input_text = processor.apply_chat_template(messages, add_generation_prompt=True)
+ inputs = processor(
+ image,
+ input_text,
+ add_special_tokens=False,
+ return_tensors="pt"
+ ).to(model.device)
+
+ output = model.generate(**inputs, max_new_tokens=50)
+ print(processor.decode(output[0]))
+
+ def test_cogvlm(self):
+ model_path = "/models/cogvlm2-llama3-chat-19B/"
+ ## test tune
+ res = os.system(
+ f"cd .. && {self.python_path} -m auto_round --mllm "
+ f"--model {model_path} --iter 2 --output_dir {self.save_dir} --device {self.device}")
+ self.assertFalse(res > 0 or res == -1, msg="cogvlm2 tuning fail")
+
+ ## test infer
+ DEVICE = f"cuda:{self.device}"
+ from transformers import AutoModelForCausalLM, AutoTokenizer
+ quantized_model_path = os.path.join(self.save_dir, "cogvlm2-llama3-chat-19B-w4g128-auto_round")
+ tokenizer = AutoTokenizer.from_pretrained(
+ quantized_model_path,
+ trust_remote_code=True
+ )
+ model = AutoModelForCausalLM.from_pretrained(
+ quantized_model_path,
+ torch_dtype="auto",
+ trust_remote_code=True,
+ device_map=DEVICE,
+ ).to(DEVICE).eval()
+
+ image_url = "https://qianwen-res.oss-cn-beijing.aliyuncs.com/Qwen-VL/assets/demo.jpeg"
+ content = "Describe this image."
+
+ text_only_template = "A chat between a curious user and an artificial intelligence assistant. The assistant gives helpful, detailed, and polite answers to the user's questions. USER: {} ASSISTANT:"
+ query = text_only_template.format(content)
+
+ image = Image.open(requests.get(image_url, stream=True).raw)
+ input_by_model = model.build_conversation_input_ids(
+ tokenizer,
+ query=query,
+ images=[image],
+ template_version='chat'
+ )
+ inputs = {
+ 'input_ids': input_by_model['input_ids'].unsqueeze(0).to(DEVICE),
+ 'token_type_ids': input_by_model['token_type_ids'].unsqueeze(0).to(DEVICE),
+ 'attention_mask': input_by_model['attention_mask'].unsqueeze(0).to(DEVICE),
+ 'images': [[input_by_model['images'][0].to(DEVICE).to(model.dtype)]] if image is not None else None,
+ }
+ gen_kwargs = {
+ "max_new_tokens": 2048,
+ "pad_token_id": 128002,
+ }
+
+ outputs = model.generate(**inputs, **gen_kwargs)
+ outputs = outputs[:, inputs['input_ids'].shape[1]:]
+ response = tokenizer.decode(outputs[0])
+ response = response.split("<|end_of_text|>")[0]
+ print(response)
+
+if __name__ == "__main__":
+ unittest.main()
\ No newline at end of file
diff --git a/test_cuda/test_vlms.py b/test_cuda/test_vlms.py
new file mode 100644
index 000000000..ad112cbce
--- /dev/null
+++ b/test_cuda/test_vlms.py
@@ -0,0 +1,147 @@
+import copy
+import shutil
+import sys
+import unittest
+import re
+
+sys.path.insert(0, "..")
+
+from PIL import Image
+import requests
+
+
+class TestAutoRound(unittest.TestCase):
+ @classmethod
+ def setUpClass(self):
+ self.save_dir = "./saved"
+
+ @classmethod
+ def tearDownClass(self):
+ shutil.rmtree(self.save_dir, ignore_errors=True)
+ shutil.rmtree("runs", ignore_errors=True)
+
+ #
+ # def test_vision_generation(self):
+ # quantized_model_path = "OPEA/Phi-3.5-vision-instruct-qvision-int4-sym-inc"
+ # from auto_round import AutoRoundConfig
+ # device = "auto" ##cpu, hpu, cuda
+ # quantization_config = AutoRoundConfig(
+ # backend=device
+ # )
+ # model = AutoModelForCausalLM.from_pretrained(quantized_model_path, trust_remote_code=True,
+ # device_map=device, quantization_config=quantization_config)
+ # tokenizer = AutoTokenizer.from_pretrained(quantized_model_path)
+ # text = "There is a girl who likes adventure,"
+ # inputs = tokenizer(text, return_tensors="pt").to(model.device)
+ # res = tokenizer.decode(model.generate(**inputs, max_new_tokens=50)[0])
+ # print(res)
+ # assert (
+ # res == """<s> There is a girl who likes adventure, and she is looking for a partner to go on a treasure hunt. She has found a map that leads to a hidden treasure, but she needs a partner to help her decipher the clues and find the treasure. You""")
+
+ def inference(self, quantized_model_dir):
+ from transformers import Qwen2VLForConditionalGeneration, AutoProcessor, AutoTokenizer
+ tokenizer = AutoTokenizer.from_pretrained(quantized_model_dir)
+ processor = AutoProcessor.from_pretrained(quantized_model_dir, trust_remote_code=True)
+ model = Qwen2VLForConditionalGeneration.from_pretrained(
+ quantized_model_dir,
+ torch_dtype="auto",
+ device_map="auto",
+ ##revision="df7f44c" ##AutoGPTQ format
+ )
+
+ image_url = "../docs/imgs/norm_bias_overview.png"
+ messages = [
+ {
+ "role": "user",
+ "content": [
+ {
+ "type": "image",
+ "image": image_url,
+ },
+ {"type": "text", "text": "Describe this image."},
+ ],
+ }
+ ]
+
+ # Preparation for inference
+ text = processor.apply_chat_template(
+ messages, tokenize=False, add_generation_prompt=True
+ )
+ image_inputs = Image.open(image_url)
+ inputs = processor(
+ text=[text],
+ images=image_inputs,
+ padding=True,
+ return_tensors="pt",
+ )
+ inputs = inputs.to(model.device)
+
+ generated_ids = model.generate(**inputs, max_new_tokens=128)
+ generated_ids_trimmed = [
+ out_ids[len(in_ids):] for in_ids, out_ids in zip(inputs.input_ids, generated_ids)
+ ]
+ output_text = processor.batch_decode(
+ generated_ids_trimmed, skip_special_tokens=True, clean_up_tokenization_spaces=False
+ )
+ print(output_text[0])
+
+ def test_vlm_tune(self):
+ from auto_round import AutoRoundMLLM
+ from transformers import Qwen2VLForConditionalGeneration, AutoProcessor, AutoTokenizer
+
+ ## load the model
+ model_name = "/models/Qwen2-VL-2B-Instruct"
+ model = Qwen2VLForConditionalGeneration.from_pretrained(
+ model_name, trust_remote_code=True)
+ tokenizer = AutoTokenizer.from_pretrained(model_name)
+ processor = AutoProcessor.from_pretrained(model_name, trust_remote_code=True)
+
+ ## quantize the model
+ bits, group_size, sym = 4, 128, True
+ autoround = AutoRoundMLLM(model, tokenizer, processor,
+ bits=bits, group_size=group_size, sym=sym, iters=1, nsamples=1)
+ autoround.quantize()
+
+ quantized_model_path = self.save_dir
+ autoround.save_quantized(quantized_model_path, format='auto_round', inplace=False)
+ self.inference(quantized_model_path)
+ shutil.rmtree(self.save_dir, ignore_errors=True)
+ autoround.save_quantized(quantized_model_path, format='auto_gptq', inplace=False)
+ self.inference(quantized_model_path)
+ shutil.rmtree(self.save_dir, ignore_errors=True)
+
+ def test_quant_not_text(self):
+ from auto_round import AutoRoundMLLM
+ from transformers import Qwen2VLForConditionalGeneration, AutoProcessor, AutoTokenizer
+
+ ## load the model
+ model_name = "/models/Qwen2-VL-2B-Instruct"
+ model = Qwen2VLForConditionalGeneration.from_pretrained(
+ model_name, trust_remote_code=True)
+ tokenizer = AutoTokenizer.from_pretrained(model_name)
+ processor = AutoProcessor.from_pretrained(model_name, trust_remote_code=True)
+
+ ## quantize the model
+ bits, group_size, sym = 4, 128, True
+ autoround = AutoRoundMLLM(model, tokenizer, processor,
+ bits=bits, group_size=group_size, sym=sym, iters=1, nsamples=1,quant_nontext_module=True)
+ autoround.quantize()
+
+ quantized_model_path = "./saved"
+ autoround.save_quantized(quantized_model_path, format='auto_round', inplace=False)
+ self.inference(quantized_model_path)
+ shutil.rmtree("./saved", ignore_errors=True)
+
+ def test_quant_not_text_fp_layers(self):
+ import os
+ python_path = sys.executable
+ absolute_path = os.path.abspath(self.save_dir)
+ res = os.system(
+ f"cd .. && {python_path} -m auto_round --mllm --model /models/Qwen2-VL-2B-Instruct --fp_layers model.layers.27,visual.blocks.29 --quant_nontext_module --iters 1 --nsamples 1 --output_dir {absolute_path}")
+ self.inference(os.path.join(absolute_path,"Qwen2-VL-2B-Instruct-w4g128-auto_round"))
+ shutil.rmtree(os.path.join(absolute_path,"Qwen2-VL-2B-Instruct-w4g128-auto_round"), ignore_errors=True)
+
+
+
+if __name__ == "__main__":
+ unittest.main()