PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度
enable llava int4 inference with autoround format (#237)
WeiweiZhang1b698213de487
diff --git a/auto_round/auto_quantizer.py b/auto_round/auto_quantizer.pyindex 65125c7d1..965402537 100644--- a/auto_round/auto_quantizer.py+++ b/auto_round/auto_quantizer.py@@ -456,7 +456,8 @@ class AutoRoundQuantizer(HfQuantizer):def _process_model_before_weight_loading(self, model: "PreTrainedModel", **kwargs):if model.__class__.main_input_name != "input_ids":- raise RuntimeError("We can only quantize pure text model.")+ logger.warning("We can only quantize pure text models and " \+ "certain types(Llava/Qwen-VL/Phi-3-vision) of multimodal models.")if self.pre_quantized:model = self.convert_model(model)@@ -485,3 +486,4 @@ if transformers_version[0] == 4 and transformers_version[1] < 38:transformers.quantizers.auto.AutoHfQuantizer = AutoHfQuantizertransformers.modeling_utils.AutoHfQuantizer = AutoHfQuantizer+diff --git a/auto_round/export/export_to_autogptq/export.py b/auto_round/export/export_to_autogptq/export.pyindex feb71bae1..836fa4e50 100644--- a/auto_round/export/export_to_autogptq/export.py+++ b/auto_round/export/export_to_autogptq/export.py@@ -152,6 +152,8 @@ def pack_layer(name, model, layer_config, backend, pbar):qlayer.pack(layer, scale, zero, None)qlayer.to(device)pbar.update(1)++@register_format("auto_gptq")def save_quantized_as_autogptq(output_dir, inplace=True, backend="auto_gptq:exllamav2",**kwargs):@@ -215,9 +217,10 @@ def save_quantized_as_autogptq(output_dir, inplace=True, backend="auto_gptq:exllif hasattr(model, "config"):model.config.quantization_config = quantization_configsave(model, output_dir, safe_serialization=safe_serialization)+ return model+-##def save(model: torch.nn.Module, save_dir: str, max_shard_size: str = "5GB", safe_serialization: bool = True):"""Save model state dict and configs.@@ -248,3 +251,4 @@ def save(model: torch.nn.Module, save_dir: str, max_shard_size: str = "5GB", saf+diff --git a/auto_round/export/export_to_autoround/export.py b/auto_round/export/export_to_autoround/export.pyindex f47a8b9b3..4bee03570 100644--- a/auto_round/export/export_to_autoround/export.py+++ b/auto_round/export/export_to_autoround/export.py@@ -294,6 +294,7 @@ def save_quantized_as_autoround(output_dir, inplace=True, backend="auto_round:exif config["bits"] > 8:modules_to_not_convert.append(name)save_awq(model, output_dir, modules_to_not_convert=modules_to_not_convert)+ return modeldef save(model: nn.Module, save_dir: str, max_shard_size: str = "5GB", safe_serialization: bool = True):@@ -360,3 +361,4 @@ def save_awq(if hasattr(model, "config") and hasattr(model.config, "quantization_config"):with open(os.path.join(save_dir, config_file), "w", encoding="utf-8") as f:json.dump(quantization_config, f, indent=2)+diff --git a/auto_round/export/export_to_awq/export.py b/auto_round/export/export_to_awq/export.pyindex 79aab82e9..042d71be7 100644--- a/auto_round/export/export_to_awq/export.py+++ b/auto_round/export/export_to_awq/export.py@@ -121,6 +121,7 @@ def save_quantized_as_autoawq(output_dir, model_path, inplace=True, **kwargs):quant_config["zero_point"] = not symsave_quantized(compressed_model, save_dir=output_dir, quant_config=quant_config)+ return compressed_modelfrom safetensors.torch import save_file@@ -225,3 +226,4 @@ def get_module_name(model, module_to_find):if module is module_to_find:return namereturn None+diff --git a/auto_round/utils.py b/auto_round/utils.pyindex f123d3b87..038e14045 100644--- a/auto_round/utils.py+++ b/auto_round/utils.py@@ -805,3 +805,4 @@ def dynamic_import_inference_linear(backend, bits, group_size, sym):else:from auto_round_extension.cuda.qlinear_tritonv2 import QuantLinearreturn QuantLinear+diff --git a/examples/multimodal-modeling/Llava/README.md b/examples/multimodal-modeling/Llava/README.mdindex 14f5a926e..97570d331 100644--- a/examples/multimodal-modeling/Llava/README.md+++ b/examples/multimodal-modeling/Llava/README.md@@ -6,7 +6,7 @@ This document presents step-by-step instructions for auto-round.In this example, we introduce an straight-forward way to execute quantization on some popular multimodal models such as LLaVA.-Please note that LLAVA quantization is currently an **experimental feature** and does not yet support inference on various devices after export.+Please note that LLAVA quantized model is currently only support inference with **auto_round** format.## InstallIf you are not using Linux, do NOT proceed, see instructions for [macOS](https://github.com/haotian-liu/LLaVA/blob/main/docs/macOS.md) and [Windows](https://github.com/haotian-liu/LLaVA/blob/main/docs/Windows.md).@@ -68,12 +68,12 @@ bash run_autoround.sh```## 4. Results-Using [COCO 2017](https://cocodataset.org/) and [LLaVA-Instruct-150K](https://huggingface.co/datasets/liuhaotian/LLaVA-Instruct-150K) datasets for quantization calibration, and TextVQA dataset for evaluation. When the vision components are not involved in quantization, it is able to achieve accuracy loss within 1%. The results for fake quantized LLava-7b are as follows:+Using [COCO 2017](https://cocodataset.org/) and [LLaVA-Instruct-150K](https://huggingface.co/datasets/liuhaotian/LLaVA-Instruct-150K) datasets for quantization calibration, and TextVQA dataset for evaluation. When the vision components are not involved in quantization, it is able to achieve accuracy loss within 1%. The results for LLava-7b are as follows:| Model | Config | Precision | Hyperparameter | Accuracy% | Relative drop || :----: | :----: | :----: | :----: | :----: | :----: || liuhaotian/llava-v1.5-7b | - | FP16 | - | 58.21 | - |-| liuhaotian/llava-v1.5-7b | W4G128 | FP16 | with vision | 56.39 | -3.13% |-| liuhaotian/llava-v1.5-7b | W4G128 | FP16 | w/o vision | 58.08 | -0.22% |+| liuhaotian/llava-v1.5-7b | W4G128 | FP16 | with vision | 56.11 | -3.60% |+| liuhaotian/llava-v1.5-7b | W4G128 | FP16 | w/o vision | 57.97 | -0.41% |## 5. Known Issuesdiff --git a/examples/multimodal-modeling/Llava/main.py b/examples/multimodal-modeling/Llava/main.pyindex 46b6b956d..78f7e7466 100644--- a/examples/multimodal-modeling/Llava/main.py+++ b/examples/multimodal-modeling/Llava/main.py@@ -1,6 +1,6 @@import argparseimport sys-+import copysys.path.insert(0, '../../..')parser = argparse.ArgumentParser()import torch@@ -61,13 +61,46 @@ class CustomDataset(Dataset): # for llava tuningdef __len__(self):return len(self.list_data_dict)-+def create_data_loader(dataset, batch_size=1, data_collator=None):assert batch_size == 1, "batch_size must be 1"data_loader = DataLoader(dataset, batch_size=batch_size, shuffle=False, collate_fn=data_collator)return data_loader+def save_tower(model, save_path, quant_vision: bool = False, max_shard_size: str = "5GB", safe_serialization: bool = True):+ if not quant_vision:+ print("Won't save vision_tower since this part was not quantized.")+ return+ ori_path = save_path+ ori_tower_name = model.get_vision_tower().vision_tower_name+ vision_tower = model.get_vision_tower().vision_tower+ save_path = f'{save_path}-vision_tower'+ os.makedirs(save_path, exist_ok=True)+ quantization_config = model.config.quantization_config+ redundant_prefix = "model.vision_tower.vision_tower."+ org_block_list = copy.deepcopy(quantization_config['quant_block_list'])+ # prepare vision_tower quantize list+ quant_block_list = [element.split(redundant_prefix)[1] if redundant_prefix in element else "" \+ for sublist in org_block_list for element in sublist]+ quant_block_list = [[element for element in quant_block_list if element != ""]]+ quantization_config['quant_block_list'] = quant_block_list+ if hasattr(vision_tower, "config"):+ from transformers import AutoProcessor+ processor = AutoProcessor.from_pretrained(ori_tower_name)+ processor.save_pretrained(save_path)+ vision_tower.config.quantization_config = quantization_config+ vision_tower.config.save_pretrained(save_path)+ vision_tower.save_pretrained(save_path, max_shard_size=max_shard_size, safe_serialization=safe_serialization)+ # prepare llava model quantize list+ quant_block_list = [element if redundant_prefix not in element else "" \+ for sublist in org_block_list for element in sublist]+ quant_block_list = [[element for element in quant_block_list if element != ""]]+ quantization_config['quant_block_list'] = quant_block_list+ model.config.mm_vision_tower = save_path+ model.config.save_pretrained(ori_path)++if __name__ == '__main__':parser.add_argument(@@ -345,20 +378,24 @@ if __name__ == '__main__':for gpu_format in gpu_formats:if "round" in gpu_format:eval_folder = f'{export_dir}-round'- autoround.save_quantized(eval_folder, format=gpu_format, use_triton=False, inplace=inplace)+ compressed_model = autoround.save_quantized(eval_folder, format=gpu_format, use_triton=False, inplace=inplace)+ save_tower(compressed_model, eval_folder, quant_vision=args.quant_vision)elif "gptq" in gpu_format:eval_folder = f'{export_dir}-gpu'- autoround.save_quantized(eval_folder, format=gpu_format, use_triton=False, inplace=inplace)-+ compressed_model = autoround.save_quantized(eval_folder, format=gpu_format, use_triton=False, inplace=inplace)+ save_tower(compressed_model, eval_folder, quant_vision=args.quant_vision)if 'xpu' in deployment_device:- autoround.save_quantized(f'{export_dir}-xpu', format="itrex_xpu", use_triton=True, inplace=inplace,+ compressed_model = autoround.save_quantized(f'{export_dir}-xpu', format="itrex_xpu", use_triton=True, inplace=inplace,compression_dtype=torch.int8, compression_dim=0, use_optimum_format=False,device="xpu")+ save_tower(compressed_model, eval_folder, quant_vision=args.quant_vision)if "cpu" in deployment_device:- autoround.save_quantized(output_dir=f'{export_dir}-cpu', format='itrex', inplace=inplace)+ compressed_model = autoround.save_quantized(output_dir=f'{export_dir}-cpu', format='itrex', inplace=inplace)+ save_tower(compressed_model, eval_folder, quant_vision=args.quant_vision)if "fake" in deployment_device:model = model.to("cpu")model.save_pretrained(output_dir)+ save_tower(model, output_dir, quant_vision=args.quant_vision)tokenizer.save_pretrained(output_dir)if eval_folder is None:eval_folder = output_dir@@ -380,5 +417,3 @@ if __name__ == '__main__':evaluator.calculate_accuracy(result_file = args.eval_result_file)--diff --git a/examples/multimodal-modeling/Llava/mm_evaluation/textvqa.py b/examples/multimodal-modeling/Llava/mm_evaluation/textvqa.pyindex 2dd384603..a3b771359 100644--- a/examples/multimodal-modeling/Llava/mm_evaluation/textvqa.py+++ b/examples/multimodal-modeling/Llava/mm_evaluation/textvqa.py@@ -174,12 +174,6 @@ class TextVQAEvaluator(object):evaluator = TextVQAAccuracyEvaluator()print('Samples: {}\nAccuracy: {:.2f}%\n'.format(len(pred_list), 100. * evaluator.eval_pred_list(pred_list)))---# results---# def eval_single(annotation_file, result_file):# experiment_name = os.path.splitext(os.path.basename(result_file))[0]# print(experiment_name)@@ -199,3 +193,72 @@ class TextVQAEvaluator(object):# print('Samples: {}\nAccuracy: {:.2f}%\n'.format(len(pred_list), 100. * evaluator.eval_pred_list(pred_list)))+if __name__ == "__main__":+ import sys+ import time+ import argparse+ from llava.mm_utils import get_model_name_from_path+ from llava.model.builder import load_pretrained_model+ from transformers import AutoConfig+ parser = argparse.ArgumentParser()+ parser.add_argument(+ "--model_name", default="liuhaotian/llava-v1.5-7b"+ )+ parser.add_argument(+ "--base_model", default=None, type=float+ )+ parser.add_argument(+ "--dataset_name", default="textvqa_val"+ )+ parser.add_argument(+ "--eval_bs", default=4,+ )+ parser.add_argument(+ "--trust_remote_code", action='store_true',+ help="Whether to enable trust_remote_code"+ )+ parser.add_argument(+ "--eval_question_file", type=str,+ default="tables/question.jsonl"+ )++ parser.add_argument(+ "--eval_image_folder", type=str+ )++ parser.add_argument(+ "--eval_result_file", type=str+ )++ parser.add_argument(+ "--eval_annotation_file", type=str+ )+ args = parser.parse_args()+ s = time.time()+ config = AutoConfig.from_pretrained(args.model_name, trust_remote_code=args.trust_remote_code)+ if hasattr(config, "quantization_config"):+ quantization_config = config.quantization_config+ if "quant_method" in quantization_config and ("auto-round" in quantization_config["quant_method"] or+ ("gptq" in quantization_config["quant_method"] and args.device == "hpu")):+ try:+ from auto_round import AutoRoundConfig+ except:+ from auto_round.auto_quantizer import AutoHfQuantizer++ model_path = args.model_name+ model_name = get_model_name_from_path(model_path)+ tokenizer, model, image_processor, _ = load_pretrained_model(model_path, model_base=args.base_model, model_name=model_name,+ torch_dtype="auto")++ evaluator = TextVQAEvaluator(+ model,+ tokenizer,+ image_processor,+ args.eval_image_folder,+ args.eval_question_file,+ args.eval_annotation_file,+ model_name = model_name+ )+ evaluator.run_evaluate(result_file = args.eval_result_file)+ evaluator.calculate_accuracy(result_file = args.eval_result_file)+ print("cost time: ", time.time() - s)diff --git a/examples/multimodal-modeling/Llava/run_autoround_on_gaudi.sh b/examples/multimodal-modeling/Llava/run_autoround_on_gaudi.shdeleted file mode 100644index 12e964853..000000000--- a/examples/multimodal-modeling/Llava/run_autoround_on_gaudi.sh+++ /dev/null@@ -1,12 +0,0 @@-#!/bin/bash-set -x-model_name=liuhaotian/llava-v1.5-7b--python3 main.py \- --model_name $model_name \- --group_size 128 \- --bits 4 \- --deployment_device "fake" \- --output_dir "./tmp_autoround"--diff --git a/examples/multimodal-modeling/requirements.txt b/examples/multimodal-modeling/requirements.txtindex 2d060638b..742f061ed 100644--- a/examples/multimodal-modeling/requirements.txt+++ b/examples/multimodal-modeling/requirements.txt@@ -1,6 +1,7 @@transformerstorchtiktoken+torchvisiontransformers_stream_generatorpeftsentencepiece@@ -11,4 +12,5 @@ protobufauto-gptqopenpyxlwandb-py-cpuinfo\ No newline at end of file+py-cpuinfo+diff --git a/examples/multimodal-modeling/run_autoround.sh b/examples/multimodal-modeling/run_autoround.shdeleted file mode 100644index 3a80b6bdc..000000000--- a/examples/multimodal-modeling/run_autoround.sh+++ /dev/null@@ -1,15 +0,0 @@-#!/bin/bash-set -x-device=0--CUDA_VISIBLE_DEVICES=$device \-python3 main.py \---model_name=liuhaotian/llava-v1.5-7b \---bits 4 \---group_size 128 \---iters 200 \---deployment_device 'autoround' \---image_folder /path/to/coco/images/train2017/ \---question_file=self_made.json \---eval-path=/path/to/textvqa_data/ \---output_dir "./tmp_autoround"\ No newline at end of file