Skip to content

PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度

enable llava int4 inference with autoround format (#237)

WeiweiZhang1b698213de487
diff --git a/auto_round/auto_quantizer.py b/auto_round/auto_quantizer.py
index 65125c7d1..965402537 100644
--- a/auto_round/auto_quantizer.py
+++ b/auto_round/auto_quantizer.py
@@ -456,7 +456,8 @@ class AutoRoundQuantizer(HfQuantizer):
def _process_model_before_weight_loading(self, model: "PreTrainedModel", **kwargs):
if model.__class__.main_input_name != "input_ids":
- raise RuntimeError("We can only quantize pure text model.")
+ logger.warning("We can only quantize pure text models and " \
+ "certain types(Llava/Qwen-VL/Phi-3-vision) of multimodal models.")
if self.pre_quantized:
model = self.convert_model(model)
@@ -485,3 +486,4 @@ if transformers_version[0] == 4 and transformers_version[1] < 38:
transformers.quantizers.auto.AutoHfQuantizer = AutoHfQuantizer
transformers.modeling_utils.AutoHfQuantizer = AutoHfQuantizer
+
diff --git a/auto_round/export/export_to_autogptq/export.py b/auto_round/export/export_to_autogptq/export.py
index feb71bae1..836fa4e50 100644
--- a/auto_round/export/export_to_autogptq/export.py
+++ b/auto_round/export/export_to_autogptq/export.py
@@ -152,6 +152,8 @@ def pack_layer(name, model, layer_config, backend, pbar):
qlayer.pack(layer, scale, zero, None)
qlayer.to(device)
pbar.update(1)
+
+
@register_format("auto_gptq")
def save_quantized_as_autogptq(output_dir, inplace=True, backend="auto_gptq:exllamav2",
**kwargs):
@@ -215,9 +217,10 @@ def save_quantized_as_autogptq(output_dir, inplace=True, backend="auto_gptq:exll
if hasattr(model, "config"):
model.config.quantization_config = quantization_config
save(model, output_dir, safe_serialization=safe_serialization)
+ return model
+
-##
def save(model: torch.nn.Module, save_dir: str, max_shard_size: str = "5GB", safe_serialization: bool = True):
"""Save model state dict and configs.
@@ -248,3 +251,4 @@ def save(model: torch.nn.Module, save_dir: str, max_shard_size: str = "5GB", saf
+
diff --git a/auto_round/export/export_to_autoround/export.py b/auto_round/export/export_to_autoround/export.py
index f47a8b9b3..4bee03570 100644
--- a/auto_round/export/export_to_autoround/export.py
+++ b/auto_round/export/export_to_autoround/export.py
@@ -294,6 +294,7 @@ def save_quantized_as_autoround(output_dir, inplace=True, backend="auto_round:ex
if config["bits"] > 8:
modules_to_not_convert.append(name)
save_awq(model, output_dir, modules_to_not_convert=modules_to_not_convert)
+ return model
def save(model: nn.Module, save_dir: str, max_shard_size: str = "5GB", safe_serialization: bool = True):
@@ -360,3 +361,4 @@ def save_awq(
if hasattr(model, "config") and hasattr(model.config, "quantization_config"):
with open(os.path.join(save_dir, config_file), "w", encoding="utf-8") as f:
json.dump(quantization_config, f, indent=2)
+
diff --git a/auto_round/export/export_to_awq/export.py b/auto_round/export/export_to_awq/export.py
index 79aab82e9..042d71be7 100644
--- a/auto_round/export/export_to_awq/export.py
+++ b/auto_round/export/export_to_awq/export.py
@@ -121,6 +121,7 @@ def save_quantized_as_autoawq(output_dir, model_path, inplace=True, **kwargs):
quant_config["zero_point"] = not sym
save_quantized(compressed_model, save_dir=output_dir, quant_config=quant_config)
+ return compressed_model
from safetensors.torch import save_file
@@ -225,3 +226,4 @@ def get_module_name(model, module_to_find):
if module is module_to_find:
return name
return None
+
diff --git a/auto_round/utils.py b/auto_round/utils.py
index f123d3b87..038e14045 100644
--- a/auto_round/utils.py
+++ b/auto_round/utils.py
@@ -805,3 +805,4 @@ def dynamic_import_inference_linear(backend, bits, group_size, sym):
else:
from auto_round_extension.cuda.qlinear_tritonv2 import QuantLinear
return QuantLinear
+
diff --git a/examples/multimodal-modeling/Llava/README.md b/examples/multimodal-modeling/Llava/README.md
index 14f5a926e..97570d331 100644
--- a/examples/multimodal-modeling/Llava/README.md
+++ b/examples/multimodal-modeling/Llava/README.md
@@ -6,7 +6,7 @@ This document presents step-by-step instructions for auto-round.
In this example, we introduce an straight-forward way to execute quantization on some popular multimodal models such as LLaVA.
-Please note that LLAVA quantization is currently an **experimental feature** and does not yet support inference on various devices after export.
+Please note that LLAVA quantized model is currently only support inference with **auto_round** format.
## Install
If you are not using Linux, do NOT proceed, see instructions for [macOS](https://github.com/haotian-liu/LLaVA/blob/main/docs/macOS.md) and [Windows](https://github.com/haotian-liu/LLaVA/blob/main/docs/Windows.md).
@@ -68,12 +68,12 @@ bash run_autoround.sh
```
## 4. Results
-Using [COCO 2017](https://cocodataset.org/) and [LLaVA-Instruct-150K](https://huggingface.co/datasets/liuhaotian/LLaVA-Instruct-150K) datasets for quantization calibration, and TextVQA dataset for evaluation. When the vision components are not involved in quantization, it is able to achieve accuracy loss within 1%. The results for fake quantized LLava-7b are as follows:
+Using [COCO 2017](https://cocodataset.org/) and [LLaVA-Instruct-150K](https://huggingface.co/datasets/liuhaotian/LLaVA-Instruct-150K) datasets for quantization calibration, and TextVQA dataset for evaluation. When the vision components are not involved in quantization, it is able to achieve accuracy loss within 1%. The results for LLava-7b are as follows:
| Model | Config | Precision | Hyperparameter | Accuracy% | Relative drop |
| :----: | :----: | :----: | :----: | :----: | :----: |
| liuhaotian/llava-v1.5-7b | - | FP16 | - | 58.21 | - |
-| liuhaotian/llava-v1.5-7b | W4G128 | FP16 | with vision | 56.39 | -3.13% |
-| liuhaotian/llava-v1.5-7b | W4G128 | FP16 | w/o vision | 58.08 | -0.22% |
+| liuhaotian/llava-v1.5-7b | W4G128 | FP16 | with vision | 56.11 | -3.60% |
+| liuhaotian/llava-v1.5-7b | W4G128 | FP16 | w/o vision | 57.97 | -0.41% |
## 5. Known Issues
diff --git a/examples/multimodal-modeling/Llava/main.py b/examples/multimodal-modeling/Llava/main.py
index 46b6b956d..78f7e7466 100644
--- a/examples/multimodal-modeling/Llava/main.py
+++ b/examples/multimodal-modeling/Llava/main.py
@@ -1,6 +1,6 @@
import argparse
import sys
-
+import copy
sys.path.insert(0, '../../..')
parser = argparse.ArgumentParser()
import torch
@@ -61,13 +61,46 @@ class CustomDataset(Dataset): # for llava tuning
def __len__(self):
return len(self.list_data_dict)
-
+
def create_data_loader(dataset, batch_size=1, data_collator=None):
assert batch_size == 1, "batch_size must be 1"
data_loader = DataLoader(dataset, batch_size=batch_size, shuffle=False, collate_fn=data_collator)
return data_loader
+def save_tower(model, save_path, quant_vision: bool = False, max_shard_size: str = "5GB", safe_serialization: bool = True):
+ if not quant_vision:
+ print("Won't save vision_tower since this part was not quantized.")
+ return
+ ori_path = save_path
+ ori_tower_name = model.get_vision_tower().vision_tower_name
+ vision_tower = model.get_vision_tower().vision_tower
+ save_path = f'{save_path}-vision_tower'
+ os.makedirs(save_path, exist_ok=True)
+ quantization_config = model.config.quantization_config
+ redundant_prefix = "model.vision_tower.vision_tower."
+ org_block_list = copy.deepcopy(quantization_config['quant_block_list'])
+ # prepare vision_tower quantize list
+ quant_block_list = [element.split(redundant_prefix)[1] if redundant_prefix in element else "" \
+ for sublist in org_block_list for element in sublist]
+ quant_block_list = [[element for element in quant_block_list if element != ""]]
+ quantization_config['quant_block_list'] = quant_block_list
+ if hasattr(vision_tower, "config"):
+ from transformers import AutoProcessor
+ processor = AutoProcessor.from_pretrained(ori_tower_name)
+ processor.save_pretrained(save_path)
+ vision_tower.config.quantization_config = quantization_config
+ vision_tower.config.save_pretrained(save_path)
+ vision_tower.save_pretrained(save_path, max_shard_size=max_shard_size, safe_serialization=safe_serialization)
+ # prepare llava model quantize list
+ quant_block_list = [element if redundant_prefix not in element else "" \
+ for sublist in org_block_list for element in sublist]
+ quant_block_list = [[element for element in quant_block_list if element != ""]]
+ quantization_config['quant_block_list'] = quant_block_list
+ model.config.mm_vision_tower = save_path
+ model.config.save_pretrained(ori_path)
+
+
if __name__ == '__main__':
parser.add_argument(
@@ -345,20 +378,24 @@ if __name__ == '__main__':
for gpu_format in gpu_formats:
if "round" in gpu_format:
eval_folder = f'{export_dir}-round'
- autoround.save_quantized(eval_folder, format=gpu_format, use_triton=False, inplace=inplace)
+ compressed_model = autoround.save_quantized(eval_folder, format=gpu_format, use_triton=False, inplace=inplace)
+ save_tower(compressed_model, eval_folder, quant_vision=args.quant_vision)
elif "gptq" in gpu_format:
eval_folder = f'{export_dir}-gpu'
- autoround.save_quantized(eval_folder, format=gpu_format, use_triton=False, inplace=inplace)
-
+ compressed_model = autoround.save_quantized(eval_folder, format=gpu_format, use_triton=False, inplace=inplace)
+ save_tower(compressed_model, eval_folder, quant_vision=args.quant_vision)
if 'xpu' in deployment_device:
- autoround.save_quantized(f'{export_dir}-xpu', format="itrex_xpu", use_triton=True, inplace=inplace,
+ compressed_model = autoround.save_quantized(f'{export_dir}-xpu', format="itrex_xpu", use_triton=True, inplace=inplace,
compression_dtype=torch.int8, compression_dim=0, use_optimum_format=False,
device="xpu")
+ save_tower(compressed_model, eval_folder, quant_vision=args.quant_vision)
if "cpu" in deployment_device:
- autoround.save_quantized(output_dir=f'{export_dir}-cpu', format='itrex', inplace=inplace)
+ compressed_model = autoround.save_quantized(output_dir=f'{export_dir}-cpu', format='itrex', inplace=inplace)
+ save_tower(compressed_model, eval_folder, quant_vision=args.quant_vision)
if "fake" in deployment_device:
model = model.to("cpu")
model.save_pretrained(output_dir)
+ save_tower(model, output_dir, quant_vision=args.quant_vision)
tokenizer.save_pretrained(output_dir)
if eval_folder is None:
eval_folder = output_dir
@@ -380,5 +417,3 @@ if __name__ == '__main__':
evaluator.calculate_accuracy(result_file = args.eval_result_file)
-
-
diff --git a/examples/multimodal-modeling/Llava/mm_evaluation/textvqa.py b/examples/multimodal-modeling/Llava/mm_evaluation/textvqa.py
index 2dd384603..a3b771359 100644
--- a/examples/multimodal-modeling/Llava/mm_evaluation/textvqa.py
+++ b/examples/multimodal-modeling/Llava/mm_evaluation/textvqa.py
@@ -174,12 +174,6 @@ class TextVQAEvaluator(object):
evaluator = TextVQAAccuracyEvaluator()
print('Samples: {}\nAccuracy: {:.2f}%\n'.format(len(pred_list), 100. * evaluator.eval_pred_list(pred_list)))
-
-
-# results
-
-
-
# def eval_single(annotation_file, result_file):
# experiment_name = os.path.splitext(os.path.basename(result_file))[0]
# print(experiment_name)
@@ -199,3 +193,72 @@ class TextVQAEvaluator(object):
# print('Samples: {}\nAccuracy: {:.2f}%\n'.format(len(pred_list), 100. * evaluator.eval_pred_list(pred_list)))
+if __name__ == "__main__":
+ import sys
+ import time
+ import argparse
+ from llava.mm_utils import get_model_name_from_path
+ from llava.model.builder import load_pretrained_model
+ from transformers import AutoConfig
+ parser = argparse.ArgumentParser()
+ parser.add_argument(
+ "--model_name", default="liuhaotian/llava-v1.5-7b"
+ )
+ parser.add_argument(
+ "--base_model", default=None, type=float
+ )
+ parser.add_argument(
+ "--dataset_name", default="textvqa_val"
+ )
+ parser.add_argument(
+ "--eval_bs", default=4,
+ )
+ parser.add_argument(
+ "--trust_remote_code", action='store_true',
+ help="Whether to enable trust_remote_code"
+ )
+ parser.add_argument(
+ "--eval_question_file", type=str,
+ default="tables/question.jsonl"
+ )
+
+ parser.add_argument(
+ "--eval_image_folder", type=str
+ )
+
+ parser.add_argument(
+ "--eval_result_file", type=str
+ )
+
+ parser.add_argument(
+ "--eval_annotation_file", type=str
+ )
+ args = parser.parse_args()
+ s = time.time()
+ config = AutoConfig.from_pretrained(args.model_name, trust_remote_code=args.trust_remote_code)
+ if hasattr(config, "quantization_config"):
+ quantization_config = config.quantization_config
+ if "quant_method" in quantization_config and ("auto-round" in quantization_config["quant_method"] or
+ ("gptq" in quantization_config["quant_method"] and args.device == "hpu")):
+ try:
+ from auto_round import AutoRoundConfig
+ except:
+ from auto_round.auto_quantizer import AutoHfQuantizer
+
+ model_path = args.model_name
+ model_name = get_model_name_from_path(model_path)
+ tokenizer, model, image_processor, _ = load_pretrained_model(model_path, model_base=args.base_model, model_name=model_name,
+ torch_dtype="auto")
+
+ evaluator = TextVQAEvaluator(
+ model,
+ tokenizer,
+ image_processor,
+ args.eval_image_folder,
+ args.eval_question_file,
+ args.eval_annotation_file,
+ model_name = model_name
+ )
+ evaluator.run_evaluate(result_file = args.eval_result_file)
+ evaluator.calculate_accuracy(result_file = args.eval_result_file)
+ print("cost time: ", time.time() - s)
diff --git a/examples/multimodal-modeling/Llava/run_autoround_on_gaudi.sh b/examples/multimodal-modeling/Llava/run_autoround_on_gaudi.sh
deleted file mode 100644
index 12e964853..000000000
--- a/examples/multimodal-modeling/Llava/run_autoround_on_gaudi.sh
+++ /dev/null
@@ -1,12 +0,0 @@
-#!/bin/bash
-set -x
-model_name=liuhaotian/llava-v1.5-7b
-
-python3 main.py \
- --model_name $model_name \
- --group_size 128 \
- --bits 4 \
- --deployment_device "fake" \
- --output_dir "./tmp_autoround"
-
-
diff --git a/examples/multimodal-modeling/requirements.txt b/examples/multimodal-modeling/requirements.txt
index 2d060638b..742f061ed 100644
--- a/examples/multimodal-modeling/requirements.txt
+++ b/examples/multimodal-modeling/requirements.txt
@@ -1,6 +1,7 @@
transformers
torch
tiktoken
+torchvision
transformers_stream_generator
peft
sentencepiece
@@ -11,4 +12,5 @@ protobuf
auto-gptq
openpyxl
wandb
-py-cpuinfo
\ No newline at end of file
+py-cpuinfo
+
diff --git a/examples/multimodal-modeling/run_autoround.sh b/examples/multimodal-modeling/run_autoround.sh
deleted file mode 100644
index 3a80b6bdc..000000000
--- a/examples/multimodal-modeling/run_autoround.sh
+++ /dev/null
@@ -1,15 +0,0 @@
-#!/bin/bash
-set -x
-device=0
-
-CUDA_VISIBLE_DEVICES=$device \
-python3 main.py \
---model_name=liuhaotian/llava-v1.5-7b \
---bits 4 \
---group_size 128 \
---iters 200 \
---deployment_device 'autoround' \
---image_folder /path/to/coco/images/train2017/ \
---question_file=self_made.json \
---eval-path=/path/to/textvqa_data/ \
---output_dir "./tmp_autoround"
\ No newline at end of file