Skip to content

PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度

rm_old_vlm_cuda_ut (#678)

Weiwei1ead9fe9f16a
diff --git a/auto_round/testing_utils.py b/auto_round/testing_utils.py
index 7a656ee3a..be856083c 100644
--- a/auto_round/testing_utils.py
+++ b/auto_round/testing_utils.py
@@ -198,9 +198,6 @@ def require_vlm_env(test_case):
# pip install git+https://github.com/haotian-liu/[email protected]
env_check &= importlib.util.find_spec("llava") is not None
- # pip install git+https://github.com/deepseek-ai/DeepSeek-VL2.git
- env_check &= importlib.util.find_spec("deepseek_vl2") is not None
-
env_check &= importlib.util.find_spec("xformers") is not None
return unittest.skipUnless(env_check, "Environment is not satisfactory")(test_case)
diff --git a/test/test_cuda/test_vllm.py b/test/test_cuda/test_vllm.py
index 88a252d26..6ad99d7a2 100644
--- a/test/test_cuda/test_vllm.py
+++ b/test/test_cuda/test_vllm.py
@@ -4,7 +4,7 @@
Validating the configuration and printing results for manual checking.
- Run `pytest tests/quantization/test_auto_round.py`.
+ Run `pytest test/test_cuda/test_vllm.py`.
"""
import pytest
diff --git a/test/test_cuda/test_vlms.py b/test/test_cuda/test_vlms.py
index 0b41655fe..15ad6ca1c 100644
--- a/test/test_cuda/test_vlms.py
+++ b/test/test_cuda/test_vlms.py
@@ -23,7 +23,7 @@ class TestAutoRound(unittest.TestCase):
shutil.rmtree(self.save_dir, ignore_errors=True)
shutil.rmtree("runs", ignore_errors=True)
- #
+
# def test_vision_generation(self):
# quantized_model_path = "OPEA/Phi-3.5-vision-instruct-qvision-int4-sym-inc"
# from auto_round import AutoRoundConfig
@@ -116,105 +116,10 @@ class TestAutoRound(unittest.TestCase):
self.qwen_inference(quantized_model_path)
shutil.rmtree(self.save_dir, ignore_errors=True)
- def phi3_infernece(self, quantized_model_dir):
- from transformers import AutoModelForCausalLM, AutoProcessor
- quantized_model_path = os.path.join(quantized_model_dir, "Phi-3.5-vision-instruct-w4g128")
- res = os.system(f"cp /models/Phi-3.5-vision-instruct/*.py {quantized_model_path}")
- model = AutoModelForCausalLM.from_pretrained(
- quantized_model_path,
- device_map="auto",
- trust_remote_code=True,
- torch_dtype="float16",
- )
- processor = AutoProcessor.from_pretrained(quantized_model_path,
- trust_remote_code=True,
- num_crops=4
- )
-
- image_url = "https://qianwen-res.oss-cn-beijing.aliyuncs.com/Qwen-VL/assets/demo.jpeg"
- content = "Describe this image."
- messages = [
- {"role": "user",
- "content": "<|image_1|>\n"+content},
- ]
-
- prompt = processor.tokenizer.apply_chat_template(
- messages,
- tokenize=False,
- add_generation_prompt=True
- )
- image_inputs = Image.open(requests.get(image_url, stream=True).raw)
- inputs = processor(prompt, image_inputs, return_tensors="pt").to(model.device)
-
- generation_args = {
- "max_new_tokens": 1000,
- "temperature": 0.0,
- "do_sample": False,
- }
-
- generate_ids = model.generate(**inputs,
- eos_token_id=processor.tokenizer.eos_token_id,
- **generation_args
- )
-
- # remove input tokens
- generate_ids = generate_ids[:, inputs['input_ids'].shape[1]:]
- response = processor.batch_decode(generate_ids,
- skip_special_tokens=True,
- clean_up_tokenization_spaces=False)[0]
-
- print(response)
-
- @require_vlm_env
- def test_quant_not_text(self):
- from auto_round import AutoRoundMLLM
- from transformers import AutoModelForCausalLM, AutoProcessor, AutoTokenizer
-
- ## load the model
- model_name = "/models/Phi-3.5-vision-instruct"
- model = AutoModelForCausalLM.from_pretrained(
- model_name, trust_remote_code=True)
- tokenizer = AutoTokenizer.from_pretrained(model_name)
- processor = AutoProcessor.from_pretrained(model_name, trust_remote_code=True)
-
- ## quantize the model
- bits, group_size, sym = 4, 128, True
- autoround = AutoRoundMLLM(model, tokenizer, processor,
- bits=bits, group_size=group_size, sym=sym, iters=1, nsamples=1,quant_nontext_module=True)
- autoround.quantize()
-
- quantized_model_path = "./saved/Phi-3.5-vision-instruct-w4g128"
- autoround.save_quantized(quantized_model_path, format='auto_round', inplace=False, safe_serialization=False)
- self.phi3_infernece(os.path.abspath(self.save_dir))
- shutil.rmtree(self.save_dir, ignore_errors=True)
-
- @require_vlm_env
- def test_quant_not_text_fp_layers(self):
- import os
- python_path = sys.executable
- absolute_path = os.path.abspath(self.save_dir)
- res = os.system(
- f"cd .. && {python_path} -m auto_round --mllm --model /models/Phi-3.5-vision-instruct "
- f"--fp_layers model.layers.27,model.vision_embed_tokens.img_processor.vision_model.encoder.layers.16 "
- f"--quant_nontext_module --iters 1 --nsamples 1 --output_dir {absolute_path}")
- self.phi3_infernece(absolute_path)
- shutil.rmtree(absolute_path, ignore_errors=True)
-
@require_vlm_env
def test_mm_block_name(self):
from auto_round.utils import get_block_names
- model_name = "/models/deepseek-vl2-tiny"
- from deepseek_vl2.models import DeepseekVLV2ForCausalLM
- model = DeepseekVLV2ForCausalLM.from_pretrained(
- model_name, trust_remote_code=True, device_map="auto")
- block_name = get_block_names(model, quant_vision=True)
- self.assertTrue(len(block_name) == 3)
- self.assertTrue(any(["projector.layers.1" not in n for n in block_name]))
- block_name = get_block_names(model, quant_vision=False)
- self.assertTrue(len(block_name) == 1)
- self.assertTrue(get_block_names(model) == block_name)
-
model_name = "/models/Llama-3.2-11B-Vision-Instruct/"
from transformers import MllamaForConditionalGeneration
model = MllamaForConditionalGeneration.from_pretrained(
@@ -228,10 +133,8 @@ class TestAutoRound(unittest.TestCase):
self.assertTrue(get_block_names(model) == block_name)
-
-
-
if __name__ == "__main__":
unittest.main()
+