PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度
rm_old_vlm_cuda_ut (#678)
Weiwei1ead9fe9f16a
diff --git a/auto_round/testing_utils.py b/auto_round/testing_utils.pyindex 7a656ee3a..be856083c 100644--- a/auto_round/testing_utils.py+++ b/auto_round/testing_utils.py@@ -198,9 +198,6 @@ def require_vlm_env(test_case):# pip install git+https://github.com/haotian-liu/[email protected]env_check &= importlib.util.find_spec("llava") is not None- # pip install git+https://github.com/deepseek-ai/DeepSeek-VL2.git- env_check &= importlib.util.find_spec("deepseek_vl2") is not None-env_check &= importlib.util.find_spec("xformers") is not Nonereturn unittest.skipUnless(env_check, "Environment is not satisfactory")(test_case)diff --git a/test/test_cuda/test_vllm.py b/test/test_cuda/test_vllm.pyindex 88a252d26..6ad99d7a2 100644--- a/test/test_cuda/test_vllm.py+++ b/test/test_cuda/test_vllm.py@@ -4,7 +4,7 @@Validating the configuration and printing results for manual checking.- Run `pytest tests/quantization/test_auto_round.py`.+ Run `pytest test/test_cuda/test_vllm.py`."""import pytestdiff --git a/test/test_cuda/test_vlms.py b/test/test_cuda/test_vlms.pyindex 0b41655fe..15ad6ca1c 100644--- a/test/test_cuda/test_vlms.py+++ b/test/test_cuda/test_vlms.py@@ -23,7 +23,7 @@ class TestAutoRound(unittest.TestCase):shutil.rmtree(self.save_dir, ignore_errors=True)shutil.rmtree("runs", ignore_errors=True)- #+# def test_vision_generation(self):# quantized_model_path = "OPEA/Phi-3.5-vision-instruct-qvision-int4-sym-inc"# from auto_round import AutoRoundConfig@@ -116,105 +116,10 @@ class TestAutoRound(unittest.TestCase):self.qwen_inference(quantized_model_path)shutil.rmtree(self.save_dir, ignore_errors=True)- def phi3_infernece(self, quantized_model_dir):- from transformers import AutoModelForCausalLM, AutoProcessor- quantized_model_path = os.path.join(quantized_model_dir, "Phi-3.5-vision-instruct-w4g128")- res = os.system(f"cp /models/Phi-3.5-vision-instruct/*.py {quantized_model_path}")- model = AutoModelForCausalLM.from_pretrained(- quantized_model_path,- device_map="auto",- trust_remote_code=True,- torch_dtype="float16",- )- processor = AutoProcessor.from_pretrained(quantized_model_path,- trust_remote_code=True,- num_crops=4- )-- image_url = "https://qianwen-res.oss-cn-beijing.aliyuncs.com/Qwen-VL/assets/demo.jpeg"- content = "Describe this image."- messages = [- {"role": "user",- "content": "<|image_1|>\n"+content},- ]-- prompt = processor.tokenizer.apply_chat_template(- messages,- tokenize=False,- add_generation_prompt=True- )- image_inputs = Image.open(requests.get(image_url, stream=True).raw)- inputs = processor(prompt, image_inputs, return_tensors="pt").to(model.device)-- generation_args = {- "max_new_tokens": 1000,- "temperature": 0.0,- "do_sample": False,- }-- generate_ids = model.generate(**inputs,- eos_token_id=processor.tokenizer.eos_token_id,- **generation_args- )-- # remove input tokens- generate_ids = generate_ids[:, inputs['input_ids'].shape[1]:]- response = processor.batch_decode(generate_ids,- skip_special_tokens=True,- clean_up_tokenization_spaces=False)[0]-- print(response)-- @require_vlm_env- def test_quant_not_text(self):- from auto_round import AutoRoundMLLM- from transformers import AutoModelForCausalLM, AutoProcessor, AutoTokenizer-- ## load the model- model_name = "/models/Phi-3.5-vision-instruct"- model = AutoModelForCausalLM.from_pretrained(- model_name, trust_remote_code=True)- tokenizer = AutoTokenizer.from_pretrained(model_name)- processor = AutoProcessor.from_pretrained(model_name, trust_remote_code=True)-- ## quantize the model- bits, group_size, sym = 4, 128, True- autoround = AutoRoundMLLM(model, tokenizer, processor,- bits=bits, group_size=group_size, sym=sym, iters=1, nsamples=1,quant_nontext_module=True)- autoround.quantize()-- quantized_model_path = "./saved/Phi-3.5-vision-instruct-w4g128"- autoround.save_quantized(quantized_model_path, format='auto_round', inplace=False, safe_serialization=False)- self.phi3_infernece(os.path.abspath(self.save_dir))- shutil.rmtree(self.save_dir, ignore_errors=True)-- @require_vlm_env- def test_quant_not_text_fp_layers(self):- import os- python_path = sys.executable- absolute_path = os.path.abspath(self.save_dir)- res = os.system(- f"cd .. && {python_path} -m auto_round --mllm --model /models/Phi-3.5-vision-instruct "- f"--fp_layers model.layers.27,model.vision_embed_tokens.img_processor.vision_model.encoder.layers.16 "- f"--quant_nontext_module --iters 1 --nsamples 1 --output_dir {absolute_path}")- self.phi3_infernece(absolute_path)- shutil.rmtree(absolute_path, ignore_errors=True)-@require_vlm_envdef test_mm_block_name(self):from auto_round.utils import get_block_names- model_name = "/models/deepseek-vl2-tiny"- from deepseek_vl2.models import DeepseekVLV2ForCausalLM- model = DeepseekVLV2ForCausalLM.from_pretrained(- model_name, trust_remote_code=True, device_map="auto")- block_name = get_block_names(model, quant_vision=True)- self.assertTrue(len(block_name) == 3)- self.assertTrue(any(["projector.layers.1" not in n for n in block_name]))- block_name = get_block_names(model, quant_vision=False)- self.assertTrue(len(block_name) == 1)- self.assertTrue(get_block_names(model) == block_name)-model_name = "/models/Llama-3.2-11B-Vision-Instruct/"from transformers import MllamaForConditionalGenerationmodel = MllamaForConditionalGeneration.from_pretrained(@@ -228,10 +133,8 @@ class TestAutoRound(unittest.TestCase):self.assertTrue(get_block_names(model) == block_name)---if __name__ == "__main__":unittest.main()+