PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度
Fix multimodal and moe issue (#191)
WeiweiZhang1ca59d36a9407
diff --git a/auto_round/autoround.py b/auto_round/autoround.pyindex 674b758bc..5b4b30a9e 100644--- a/auto_round/autoround.py+++ b/auto_round/autoround.py@@ -68,8 +68,8 @@ class AutoRound(object):'sym': False'act_data_type': None,'act_bits': 32,- 'group_size': None,- 'sym': None,+ 'act_group_size': None,+ 'act_sym': None,}...@@ -99,12 +99,11 @@ class AutoRound(object):data_type (str): The data type to be used (default is "int").scale_dtype (str): The data type of quantization scale to be used (default is "float16"), different kernelshave different choices.- multimodal(bool): Enable multimodal model quantization, (default is "False").act_bits (int): Number of bits for activation quantization. Default is 32.act_group_size (int): Group size for activation quantization. Default is None.act_sym (bool): Whether to use symmetric activation quantization. Default is None.act_dynamic (bool): Whether to use dynamic activation quantization. Default is True.-+ quant_block_list (list): A list whose elements are list of block's layer names to be quantized.Returns:The quantized model."""@@ -140,11 +139,11 @@ class AutoRound(object):dynamic_max_gap: int = -1,data_type: str = "int",scale_dtype: str = "fp16",- multimodal:bool = False,act_bits: int = 32,act_group_size: int = None,act_sym: bool = None,act_dynamic: bool = True,+ quant_block_list: list = None,**kwargs,):self.quantized = False@@ -176,7 +175,7 @@ class AutoRound(object):self.dataset = datasetself.iters = iters- self.multimodal = multimodal+ self.quant_block_list = quant_block_listif self.iters <= 0:logger.warning("iters must be positive, reset it to 200")self.iters = 200@@ -235,7 +234,10 @@ class AutoRound(object):The quantized model and layer configurations."""# logger.info("cache block input")- all_blocks = get_block_names(self.model, self.multimodal)+ if bool(self.quant_block_list):+ all_blocks = self.quant_block_list+ else:+ all_blocks = get_block_names(self.model)if len(all_blocks) == 0:logger.warning("could not find blocks, exit with original model")@@ -248,6 +250,7 @@ class AutoRound(object):self.start_time = time.time()all_first_block_names = [block[0] for block in all_blocks]all_inputs = self.try_cache_inter_data_gpucpu(all_first_block_names, self.nsamples, layer_names=layer_names)+ self.model = mv_module_from_gpu(self.model, self.low_cpu_mem_usage)for block_names in all_blocks:inputs = all_inputs[block_names[0]]all_inputs.pop(block_names[0])@@ -258,17 +261,16 @@ class AutoRound(object):self.n_samples = total_samplesif total_samples < self.train_bs:self.train_bs = total_samples- logger.warning(f"force the train batch size to {total_samples} ")+ logger.warning(f"force the train batch size to {total_samples}")- self.model = mv_module_from_gpu(self.model, self.low_cpu_mem_usage)- torch.cuda.empty_cache()- self.quant_blocks(- self.model,- inputs,- block_names,- nblocks=self.nblocks,- device=self.device,- )+ torch.cuda.empty_cache()+ self.quant_blocks(+ self.model,+ inputs,+ block_names,+ nblocks=self.nblocks,+ device=self.device,+ )self.quant_layers(layer_names, all_inputs)@@ -366,7 +368,7 @@ class AutoRound(object):Returns:None"""- layers_in_blocks = get_layer_names_in_block(self.model, self.supported_types, self.multimodal)+ layers_in_blocks = get_layer_names_in_block(self.model, self.supported_types, self.quant_block_list)keys = ["data_type", "bits", "group_size", "sym", "scale_dtype", "act_bits", "act_group_size", "act_sym","act_dynamic"]for n, m in self.model.named_modules():@@ -1132,7 +1134,7 @@ class AutoRound(object):data_type=self.data_type,serialization_dict=serialization_dict,backend=backend,- multimodal=self.multimodal,+ quant_block_list=self.quant_block_list,**kwargs)return compressed_model@@ -1147,7 +1149,7 @@ class AutoRound(object):return []layer_names = []- all_layers_in_block = get_layer_names_in_block(self.model, self.supported_types, self.multimodal)+ all_layers_in_block = get_layer_names_in_block(self.model, self.supported_types, self.quant_block_list)for key in self.layer_config.keys():if key in all_layers_in_block:@@ -1518,3 +1520,4 @@ class AutoAdamRound(AutoOPTRound):)+diff --git a/auto_round/export/export_to_autogptq/export.py b/auto_round/export/export_to_autogptq/export.pyindex ddc9b0c52..ffe106b86 100644--- a/auto_round/export/export_to_autogptq/export.py+++ b/auto_round/export/export_to_autogptq/export.py@@ -36,8 +36,8 @@# SOFTWARE.import torch-from auto_round.utils import check_to_quantized, get_block_names, get_module, logger, get_layer_names_in_block, \- set_module+from auto_round.utils import check_to_quantized, get_block_names,\+ get_module, logger, get_layer_names_in_block, set_moduleimport copyimport jsonimport os@@ -107,12 +107,16 @@ def save_quantized_as_autogptq(output_dir, inplace=True, backend="auto_gptq:exllmodel = kwargs["model"]tokenizer = kwargs["tokenizer"]supported_types = kwargs["supported_types"]-+ safe_serialization = True if 'safe_serialization' not in kwargs.keys() else kwargs["safe_serialization"]+ quant_block_list = kwargs["quant_block_list"]logger.info("Saving quantized model to autogptq format, this may take a while...")if tokenizer is not None:tokenizer.save_pretrained(output_dir)##check module quantized in block, this may have bug for mixed precision quantization- all_blocks = get_block_names(model, multimodal=True)+ if bool(quant_block_list):+ all_blocks = quant_block_list+ else:+ all_blocks = get_block_names(model)all_to_quantized = Truemodules_in_block_to_quantize = []for block_names in all_blocks:@@ -201,7 +205,7 @@ def save_quantized_as_autogptq(output_dir, inplace=True, backend="auto_gptq:exllquantization_config["modules_in_block_to_quantize"] = modules_in_block_to_quantizeif hasattr(model, "config"):model.config.quantization_config = quantization_config- save(model, output_dir)+ save(model, output_dir, safe_serialization=safe_serialization)##@@ -233,3 +237,4 @@ def save(model: torch.nn.Module, save_dir: str, max_shard_size: str = "5GB", safwith open(os.path.join(save_dir, config_file), "w", encoding="utf-8") as f:json.dump(model.config.quantization_config, f, indent=2)+diff --git a/auto_round/export/export_to_autoround/export.py b/auto_round/export/export_to_autoround/export.pyindex 13b153317..309aba660 100644--- a/auto_round/export/export_to_autoround/export.py+++ b/auto_round/export/export_to_autoround/export.py@@ -165,11 +165,12 @@ def save_quantized_as_autoround(output_dir, inplace=True, backend="auto_round:exbackend = backend.replace("auto_round", "auto_gptq")model = kwargs["model"]- multimodal = kwargs["multimodal"]+ quant_block_list = kwargs["quant_block_list"]+ safe_serialization = True if 'safe_serialization' not in kwargs.keys() else kwargs["safe_serialization"]model = model.to(torch.float16) ##force to fp16if not inplace:model = copy.deepcopy(model.to("cpu"))- layer_names_in_block = get_layer_names_in_block(model, multimodal=multimodal)+ layer_names_in_block = get_layer_names_in_block(model, quant_block_list=quant_block_list)layer_config = kwargs["layer_config"]quantization_config = kwargs["serialization_dict"]@@ -199,7 +200,6 @@ def save_quantized_as_autoround(output_dir, inplace=True, backend="auto_round:exquantization_config["extra_config"] = extra_configwith tctl.threadpool_limits(limits=1):for name in layer_config.keys():-config = kwargs["layer_config"][name]if config["bits"] > 8:continue@@ -251,7 +251,7 @@ def save_quantized_as_autoround(output_dir, inplace=True, backend="auto_round:extokenizer = kwargs["tokenizer"]if tokenizer is not None:tokenizer.save_pretrained(output_dir)- save(model, output_dir)+ save(model, output_dir, safe_serialization=safe_serialization)def save(model: nn.Module, save_dir: str, max_shard_size: str = "5GB", safe_serialization: bool = True):@@ -281,3 +281,4 @@ def save(model: nn.Module, save_dir: str, max_shard_size: str = "5GB", safe_seriwith open(os.path.join(save_dir, config_file), "w", encoding="utf-8") as f:json.dump(model.config.quantization_config, f, indent=2)+diff --git a/auto_round/export/export_to_itrex/export.py b/auto_round/export/export_to_itrex/export.pyindex 50680558e..1adf65987 100644--- a/auto_round/export/export_to_itrex/export.py+++ b/auto_round/export/export_to_itrex/export.py@@ -75,6 +75,7 @@ def save_quantized_as_itrex(output_dir, inplace=True, **kwargs):enable_quanted_input = kwargs["enable_quanted_input"]scale_dtype = kwargs["scale_dtype"]tokenizer = kwargs["tokenizer"]+ safe_serialization = True if 'safe_serialization' not in kwargs.keys() else kwargs["safe_serialization"]compressed_model = pack_model(model, layer_config, inplace=inplace)if output_dir is None:@@ -97,7 +98,7 @@ def save_quantized_as_itrex(output_dir, inplace=True, **kwargs):config.save_pretrained(output_dir)quantize_config.save_pretrained(output_dir)try:- compressed_model.save_pretrained(output_dir, safe_serialization=True)+ compressed_model.save_pretrained(output_dir, safe_serialization=safe_serialization)if tokenizer is not None:tokenizer.save_pretrained(output_dir)logger.info("Saved config file and weights of quantized model to {}.".format(output_dir))@@ -203,9 +204,9 @@ def pack_model(else:q_config = layer_configfor k, v in q_config.items():- logger.info(f"Packing {k}")if "float" in v["data_type"]:continue+ logger.info(f"Packing {k}")dtype = v["data_type"]num_bits = v["bits"]group_size = v["group_size"]@@ -259,3 +260,4 @@ def pack_model(new_module.pack(int_weight, scale, zp, m.bias)set_module(compressed_model, k, new_module)return compressed_model+diff --git a/auto_round/special_model_handler.py b/auto_round/special_model_handler.pyindex 86fcb8260..ce79cfb2a 100644--- a/auto_round/special_model_handler.py+++ b/auto_round/special_model_handler.py@@ -12,12 +12,11 @@# See the License for the specific language governing permissions and# limitations under the License.-from .utils import logger, torch+import torchshare_attention_mask_tuple = ("baichuan",)special_states_dim_tuple = ("chatglm",)-not_share_position_ids_tuple = ("llava","phi3_v")-+not_share_position_ids_tuple = ("llava", "phi3_v",)def check_share_attention_mask(model, hidden_states, attention_mask=None, **kwargs):"""Checks if the attention mask states of the hidden states are shared in the model.@@ -65,3 +64,4 @@ def check_not_share_position_ids(model, **kwargs):breakreturn bool(is_special)+diff --git a/auto_round/utils.py b/auto_round/utils.pyindex f7bdd1f27..2d6bd52fa 100644--- a/auto_round/utils.py+++ b/auto_round/utils.py@@ -27,7 +27,6 @@ from torch.amp import autocastfrom functools import lru_cache-@lru_cache(None)def warning_once(self, msg: str):self.warning(msg)@@ -166,6 +165,7 @@ def to_device(input, device=torch.device("cpu")):return input+def mv_module_from_gpu(module, low_cpu_mem_usage=False):"""Moves module from gpu to cpu or meta if low_cpu_mem_usage is true.@@ -232,22 +232,76 @@ def check_is_cpu(device):return device == torch.device("cpu") or device == "cpu"-def get_block_names(model, multimodal=False):+def validate_modules(module_names):+ """+ Test a list of modules' validity.++ Args:+ modules (list of str): List of strings to be validated.++ Returns:+ bool: True if all modules have equal length or not dependent, otherwise False.+ """+ if not bool(module_names): # pragma: no cover+ raise ValueError(f"Empty modules")+ if len(module_names) < 2:+ return True+ split_modules = [s.split('.') for s,_ in module_names]+ lengths = [len(parts) for parts in split_modules]+ if len(set(lengths)) == 1: # pragma: no cover+ return True+ max_length = max(lengths)+ min_length = min(lengths)+ longest_module = next(s for s in split_modules if len(s) == max_length)+ shortest_module = next(s for s in split_modules if len(s) == min_length)+ shortest_module = '.'.join(shortest_module)+ longest_module = '.'.join(longest_module)+ # Check if the shortest name is a substring of the longest name+ if shortest_module in longest_module: # pragma: no cover+ raise ValueError(f"Invalid modules, at least two modules detected"\+ " as dependent, {shortest_module} and {longest_module}")+ return True+++def get_block_names(model):"""Get the block names for transformers-like networks.Args:model: The model.Returns:- block_names: A list of block names.+ block_names: A list whose elements are list of block's layer names+ """+ block_names = []+ target_modules = []+ for n, m in model.named_modules():+ if hasattr(type(m), "__name__") and "ModuleList" in type(m).__name__:+ target_modules.append((n, m))+ break ## only find the first modulelist, may be not robust+ for i,target_m in enumerate(target_modules):+ block_names.append([])+ for n, m in target_m[1].named_children():+ block_names[i].append(target_m[0] + "." + n)+ return block_names+++def get_multimodal_block_names(model, quant_vision=False):+ """Get the multimodal model block names for transformers-like networks.++ Args:+ model: The model.++ Returns:+ block_names: A list whose elements are list of block's layer names"""block_names = []target_modules = []+ Vison_blocks_tuple = ("vision", "visual",)for n, m in model.named_modules():- if hasattr(type(m), "__name__") and "ModuleList" in type(m).__name__ \- and (multimodal or ('vision' not in n and 'visual' not in n)):- target_modules.append((n, m))- # break ## only find the first modulelist, may be not robust+ if hasattr(type(m), "__name__") and "ModuleList" in type(m).__name__:+ if quant_vision or all(key not in n.lower() for key in (Vison_blocks_tuple)):+ target_modules.append((n, m))+ validate_modules(target_modules)for i,target_m in enumerate(target_modules):block_names.append([])for n, m in target_m[1].named_children():@@ -607,7 +661,7 @@ def check_memory_availability(device, inputs, weight, org_seqlen, org_bs):def get_layer_names_in_block(model, supported_types=[torch.nn.Linear,- transformers.modeling_utils.Conv1D], multimodal=False):+ transformers.modeling_utils.Conv1D], quant_block_list=None):"""Retrieves the names of layers within each block of the model.Returns:@@ -618,7 +672,10 @@ def get_layer_names_in_block(model, supported_types=[torch.nn.Linear,if isinstance(m, tuple(supported_types)):m.tmp_name = nlayers_in_block = []- all_blocks = get_block_names(model, multimodal)+ if bool(quant_block_list):+ all_blocks = quant_block_list+ else:+ all_blocks = get_block_names(model)for block_names in all_blocks:for block_name in block_names:block = get_module(model, block_name)@@ -742,3 +799,4 @@ def dynamic_import_inference_linear(backend, bits, group_size, sym):return QuantLinear+diff --git a/examples/language-modeling/README.md b/examples/language-modeling/README.mdindex bc3fec102..f9fb00598 100644--- a/examples/language-modeling/README.md+++ b/examples/language-modeling/README.md@@ -147,28 +147,32 @@ The transformers version required varies across different types of models. Here,| Model | Transformers version || :----: | :----: |-| EleutherAI/gpt-j-6b | 4.28/4.30/4.34/4.36 |-| huggyllama/llama-7b | 4.28/4.30/4.34/4.36 |-| meta-llama/Llama-2-7b-hf | 4.30/4.34/4.36 |-| facebook/opt-6.7b | 4.28/4.30/4.34/4.36 |-| tiiuae/falcon-7b | 4.28/4.30/4.34/4.36 |+| EleutherAI/gpt-j-6b | 4.28/4.30/4.34/4.36/4.40 |+| huggyllama/llama-7b | 4.28/4.30/4.34/4.36/4.40 |+| meta-llama/Llama-2-7b-hf | 4.30/4.34/4.36/4.40 |+| facebook/opt-6.7b | 4.28/4.30/4.34/4.36/4.40 |+| tiiuae/falcon-7b | 4.28/4.30/4.34/4.36/4.40 || mosaicml/mpt-7b | 4.28/4.30/4.34/4.36 || mosaicml/mpt-7b-chat | 4.34 |-| bigscience/bloom-7b1 | 4.28/4.30/4.34/4.36 |-| baichuan-inc/Baichuan2-7B-Chat | 4.36 |-| Qwen/Qwen-7B | 4.28/4.30/4.34/4.36 |-| Qwen/Qwen1.5-7B-Chat | 4.38/4.40 |-| THUDM/chatglm3-6b | 4.34/4.36 |-| mistralai/Mistral-7B-v0.1 | 4.34/4.36 |-| MBZUAI/LaMini-GPT-124M | 4.34/4.36 |-| EleutherAI/gpt-neo-125m | 4.34 |-| databricks/dolly-v2-3b | 4.34 |-| stabilityai/stablelm-base-alpha-3b | 4.34 |-| Intel/neural-chat-7b-v3 | 4.34/4.36 |-| rinna/bilingual-gpt-neox-4b | 4.36 |-| microsoft/phi-2 | 4.36 |+| bigscience/bloom-7b1 | 4.28/4.30/4.34/4.36/4.40 |+| baichuan-inc/Baichuan2-7B-Chat | 4.36/4.40 |+| Qwen/Qwen-7B | 4.28/4.30/4.34/4.36/4.40 |+| Qwen/Qwen1.5-7B-Chat | 4.38/4.40/4.40 |+| THUDM/chatglm3-6b | 4.34/4.36/4.40 |+| mistralai/Mistral-7B-v0.1 | 4.34/4.36/4.40 |+| mistralai/Mixtral-8x7B-Instruct-v0.1 | 4.40 |+| MBZUAI/LaMini-GPT-124M | 4.34/4.36/4.40 |+| EleutherAI/gpt-neo-125m | 4.34/4.40 |+| databricks/dolly-v2-3b | 4.34/4.40 |+| stabilityai/stablelm-base-alpha-3b | 4.34/4.40 |+| Intel/neural-chat-7b-v3 | 4.34/4.36/4.40 |+| rinna/bilingual-gpt-neox-4b | 4.36/4.40 |+| microsoft/phi-2 | 4.36/4.40 || google/gemma-7b | 4.38/4.40 |-| Salesforce/codegen25-7b-multi | 4.33.2|+| Salesforce/codegen25-7b-multi | 4.33.2 |+++## Reference@@ -188,3 +192,4 @@ If you find SignRound useful for your research, please cite our paper:+diff --git a/examples/language-modeling/main.py b/examples/language-modeling/main.pyindex 3cd5e660d..112d8fc63 100644--- a/examples/language-modeling/main.py+++ b/examples/language-modeling/main.py@@ -396,7 +396,8 @@ if __name__ == '__main__':print(excel_name, flush=True)eval_model(model_path=output_dir, tasks=tasks, dtype=dtype, limit=None,eval_bs=args.eval_bs, use_accelerate=args.low_gpu_mem_usage,- device=torch_device, excel_file=excel_name)+ device=torch_device, excel_file=excel_name,+ trust_remote_code=not args.disable_trust_remote_code)if not args.disable_eval and lm_eval_version == "0.4.2":from eval_042.evaluation import simple_evaluate@@ -407,6 +408,7 @@ if __name__ == '__main__':model_args = f"pretrained={eval_folder}"else:exit() ## does not support cpu,xpu model eval+ model_args = model_args + f",trust_remote_code={not args.disable_trust_remote_code}"user_model = Noneif args.act_bits <= 8:user_model = model.to(device_str)@@ -418,3 +420,4 @@ if __name__ == '__main__':print(make_table(res))+diff --git a/examples/multimodal-modeling/Llava/main.py b/examples/multimodal-modeling/Llava/main.pyindex 5dc734d2b..46b6b956d 100644--- a/examples/multimodal-modeling/Llava/main.py+++ b/examples/multimodal-modeling/Llava/main.py@@ -31,7 +31,7 @@ class CustomDataset(Dataset): # for llava tuningself.tokenizer = tokenizerself.image_processor = image_processorself.args = args- self.args.is_multimodal = args.do_multimodal+ self.args.is_multimodal = args.is_multimodaldef __getitem__(self, index):sources = self.list_data_dict[index]@@ -158,8 +158,11 @@ if __name__ == '__main__':parser.add_argument("--act_bits", default=32, type=int,help="activation bits")- parser.add_argument("--do_multimodal", action='store_true',- help="To determine whether the preprocessing should handle multimodal component.")+ parser.add_argument("--is_multimodal", type=bool, default=True,+ help="To determine whether the preprocessing should handle multimodal infomations.")++ parser.add_argument("--quant_vision", action='store_true',+ help="To determine whether the quantization should handle vision component.")# ========== Calibration Datasets =============parser.add_argument("--mm-use-im-start-end", type=bool, default=False)@@ -170,9 +173,6 @@ if __name__ == '__main__':parser.add_argument("--question_file", default=None, type=str,help="The dataset for quantization training. It can be a custom one.")- # parser.add_argument("--dataset", default=None, type=str,- # help="The dataset for quantization training. It can be a custom one.")-# ================= Evaluation Related =====================parser.add_argument("--eval-question-file", type=str, default="tables/question.jsonl")@@ -215,6 +215,7 @@ if __name__ == '__main__':from auto_round import (AutoRound,AutoAdamRound)+ from auto_round.utils import get_multimodal_block_namesmodel = model.eval()@@ -305,15 +306,17 @@ if __name__ == '__main__':if args.quant_lm_head and args.low_gpu_mem_usage:print(f"warning, low_gpu_mem_usage=False is strongly recommended if the whole model could be loaded to "f"gpu")++ quant_block_list = get_multimodal_block_names(model, args.quant_vision)autoround = round(model, tokenizer, args.bits, args.group_size, sym=args.sym, batch_size=args.train_bs,dataset=dataloader, seqlen=seqlen, nblocks=args.nblocks, iters=args.iters, lr=args.lr,minmax_lr=args.minmax_lr, enable_quanted_input=not args.disable_quanted_input, device=device_str,- amp=not args.disable_amp, nsamples=args.nsamples,- low_gpu_mem_usage=args.low_gpu_mem_usage,+ amp=not args.disable_amp, nsamples=args.nsamples, layer_config=layer_config,+ low_gpu_mem_usage=args.low_gpu_mem_usage, scale_dtype=args.scale_dtype,seed=args.seed, gradient_accumulate_steps=args.gradient_accumulate_steps,- scale_dtype=args.scale_dtype, layer_config=layer_config,- enable_minmax_tuning=not args.disable_minmax_tuning, act_bits=args.act_bits, multimodal=args.do_multimodal)+ enable_minmax_tuning=not args.disable_minmax_tuning, act_bits=args.act_bits,+ quant_block_list=quant_block_list)model, _ = autoround.quantize()model_name = args.model_name.rstrip("/")@@ -378,3 +381,4 @@ if __name__ == '__main__':+diff --git a/examples/multimodal-modeling/Llava/run_autoround.sh b/examples/multimodal-modeling/Llava/run_autoround.shindex 7f888de53..1d5e39053 100644--- a/examples/multimodal-modeling/Llava/run_autoround.sh+++ b/examples/multimodal-modeling/Llava/run_autoround.sh@@ -17,3 +17,4 @@ python3 main.py \--eval-annotation-file /path/to/textvqa/TextVQA_0.5.1_val.json \--eval-result-file "./tmp_autoround" \--output_dir "./tmp_autoround"+diff --git a/examples/multimodal-modeling/Qwen-VL/main.py b/examples/multimodal-modeling/Qwen-VL/main.pyindex 4a49ab728..28544271a 100644--- a/examples/multimodal-modeling/Qwen-VL/main.py+++ b/examples/multimodal-modeling/Qwen-VL/main.py@@ -290,8 +290,8 @@ if __name__ == '__main__':parser.add_argument("--act_bits", default=32, type=int,help="activation bits")- parser.add_argument("--do_multimodal", action='store_true',- help="To determine whether the preprocessing should handle multimodal component.")+ parser.add_argument("--quant_vision", action='store_true',+ help="To determine whether the quantization should handle vision component.")# ========== Calibration Datasets =============parser.add_argument("--image_folder", default="coco", type=str,@@ -371,6 +371,7 @@ if __name__ == '__main__':from auto_round import (AutoRound,AutoAdamRound)+ from auto_round.utils import get_multimodal_block_names# model = model.eval()seqlen = args.seqlen@@ -439,15 +440,18 @@ if __name__ == '__main__':if args.quant_lm_head and args.low_gpu_mem_usage:print(f"warning, low_gpu_mem_usage=False is strongly recommended if the whole model could be loaded to "f"gpu")++ quant_block_list = get_multimodal_block_names(model, args.quant_vision)autoround = round(model, tokenizer, args.bits, args.group_size, sym=args.sym, batch_size=args.train_bs,dataset=dataloader, seqlen=seqlen, nblocks=args.nblocks, iters=args.iters, lr=args.lr,- minmax_lr=args.minmax_lr, enable_quanted_input=not args.disable_quanted_input, device=device_str,+ minmax_lr=args.minmax_lr, enable_quanted_input=not args.disable_quanted_input,amp=not args.disable_amp, nsamples=args.nsamples,- low_gpu_mem_usage=args.low_gpu_mem_usage,+ low_gpu_mem_usage=args.low_gpu_mem_usage, device=device_str,seed=args.seed, gradient_accumulate_steps=args.gradient_accumulate_steps,scale_dtype=args.scale_dtype, layer_config=layer_config,- enable_minmax_tuning=not args.disable_minmax_tuning, act_bits=args.act_bits, multimodal=args.do_multimodal)+ enable_minmax_tuning=not args.disable_minmax_tuning, act_bits=args.act_bits,+ quant_block_list=quant_block_list)model, _ = autoround.quantize()model_name = args.model_name.rstrip("/")@@ -520,3 +524,4 @@ if __name__ == '__main__':device=str(torch_device))+diff --git a/examples/multimodal-modeling/Qwen-VL/run_autoround.sh b/examples/multimodal-modeling/Qwen-VL/run_autoround.shindex 5845fe302..d59ad510d 100644--- a/examples/multimodal-modeling/Qwen-VL/run_autoround.sh+++ b/examples/multimodal-modeling/Qwen-VL/run_autoround.sh@@ -2,6 +2,8 @@set -xdevice=0+# --quant_vision ## for vision quantization+CUDA_VISIBLE_DEVICES=$device \python3 main.py \--model_name=Qwen/Qwen-VL \@@ -9,9 +11,9 @@ python3 main.py \--group_size 128 \--iters 200 \--seqlen 512 \---do_multimodal \--disable_quanted_input \--deployment_device 'auto_round' \--image_folder /path/to/coco/images/train2017/ \--question_file /path/to/Qwen-VL_mix665k.json \--output_dir "./tmp_autoround"+diff --git a/test/test_block_names.py b/test/test_block_names.pynew file mode 100644index 000000000..5b6218a51--- /dev/null+++ b/test/test_block_names.py@@ -0,0 +1,179 @@+import copy+import shutil+import sys+import unittest+sys.path.insert(0, ".")+sys.path.insert(0, "..")+import torch+import torch.nn as nn+from transformers import AutoModelForCausalLM, AutoTokenizer+from auto_round import AutoRound++class LLMDataLoader:+ def __init__(self, input_size=10):+ self.batch_size = 1+ self.input_size = input_size+ def __iter__(self):+ for i in range(2):+ yield torch.ones([1, self.input_size], dtype=torch.long)++++# ================= simple multimodal model =================+class TextEncoder(nn.Module):+ def __init__(self, input_size, hidden_size):+ super(TextEncoder, self).__init__()+ self.fc = nn.Linear(input_size, hidden_size)++ def forward(self, x):+ return self.fc(x)++class VisionEncoder(nn.Module):+ def __init__(self, input_size, hidden_size):+ super(VisionEncoder, self).__init__()+ self.fc = nn.Linear(input_size, hidden_size)++ def forward(self, x):+ return self.fc(x)++class TextEncoderModuleList(nn.ModuleList):+ pass++class VisionEncoderModuleList(nn.ModuleList):+ pass++class SimpleMultimodalModel(nn.Module):+ def __init__(self, text_input_size, image_input_size, hidden_size, num_text_encoders, num_image_encoders):+ super(SimpleMultimodalModel, self).__init__()+ self.text_encoders = TextEncoderModuleList([TextEncoder(text_input_size, hidden_size) for _ in range(num_text_encoders)])+ self.vision_encoders = VisionEncoderModuleList([VisionEncoder(image_input_size, hidden_size) for _ in range(num_image_encoders)])+ self.classifier = nn.Linear(hidden_size * 2, 1) # Binary classification task++ def forward(self, text_input, image_input):+ text_features = [encoder(text_input) for encoder in self.text_encoders]+ image_features = [encoder(image_input) for encoder in self.vision_encoders]++ # Summing the outputs of all text encoders and image encoders+ combined_text_features = torch.stack(text_features, dim=1).sum(dim=1)+ combined_image_features = torch.stack(image_features, dim=1).sum(dim=1)++ combined_features = torch.cat((combined_text_features, combined_image_features), dim=1)+ output = self.classifier(combined_features)+ return output+++# ================= simple MoE model =================+class Expert(nn.Module):+ def __init__(self, input_size, hidden_size):+ super(Expert, self).__init__()+ self.fc = nn.Linear(input_size, hidden_size)++ def forward(self, x):+ return self.fc(x)++class ExpertGroupModuleList(nn.ModuleList):+ pass++class ExpertModuleList(nn.ModuleList):+ pass++class NestedMoEModel(nn.Module):+ def __init__(self, input_size, hidden_size, num_groups, experts_per_group):+ super(NestedMoEModel, self).__init__()+ self.expert_groups = ExpertModuleList([+ ExpertGroupModuleList([Expert(input_size, hidden_size) for _ in range(experts_per_group)])+ for _ in range(num_groups)+ ])+ self.gate = nn.Linear(input_size, num_groups * experts_per_group)++ def forward(self, x):+ gate_outputs = torch.softmax(self.gate(x), dim=1)+ expert_outputs = []++ for group in self.expert_groups:+ group_outputs = [expert(x) for expert in group]+ group_output_sum = torch.stack(group_outputs, dim=1).sum(dim=1)+ expert_outputs.append(group_output_sum)++ expert_outputs = torch.stack(expert_outputs, dim=1).view(x.size(0), -1)+ output = torch.sum(gate_outputs.unsqueeze(2) * expert_outputs.unsqueeze(1), dim=1)+ return output+++class TestQuantizationBlocks(unittest.TestCase):+ @classmethod+ def setUpClass(self):+ self.model_name = "MBZUAI/LaMini-GPT-124M"+ self.tokenizer = AutoTokenizer.from_pretrained(self.model_name, trust_remote_code=True)+ self.llm_dataloader = LLMDataLoader()++ @classmethod+ def tearDownClass(self):+ shutil.rmtree("./saved", ignore_errors=True)+ shutil.rmtree("runs", ignore_errors=True)+++ def test_moe_quant(self):+ input_size = 10+ hidden_size = 10+ num_groups = 2+ experts_per_group = 2+ self.llm_dataloader = LLMDataLoader(input_size)+ self.model = NestedMoEModel(input_size, hidden_size, num_groups, experts_per_group)+ from auto_round.utils import get_multimodal_block_names, get_block_names+ llm_block_names = get_block_names(self.model)+ all_block_names = []+ try:+ all_block_names = get_multimodal_block_names(self.model, quant_vision=True)+ except:+ pass+ assert len(llm_block_names) != len(all_block_names)+++ def test_multimodal_quant(self):+ num_text_encoders = 1+ num_image_encoders = 1+ image_input_size = 10+ text_input_size = 10+ hidden_size = 10+ self.model = SimpleMultimodalModel(text_input_size, image_input_size, hidden_size, num_text_encoders, num_image_encoders)+ from auto_round.utils import get_multimodal_block_names, get_block_names+ llm_block_names = get_block_names(self.model)+ block_names_wo_vision = get_multimodal_block_names(self.model, quant_vision=False)+ block_names_with_vision = get_multimodal_block_names(self.model, quant_vision=True)+ assert block_names_wo_vision == llm_block_names+ assert len(block_names_wo_vision) != (block_names_with_vision)+++ def test_block_name_quant(self):+ self.model = AutoModelForCausalLM.from_pretrained(self.model_name, torch_dtype="auto", trust_remote_code=True)+ from auto_round.utils import get_block_names, validate_modules+ llm_block_names = get_block_names(self.model)+ validate_modules(llm_block_names)+ bits, group_size, sym, train_bs = 4, 128, False, 20+ autoround = AutoRound(+ self.model,+ self.tokenizer,+ bits=bits,+ group_size=group_size,+ sym=sym,+ iters=2,+ seqlen=2,+ train_bs=train_bs,+ dataset=self.llm_dataloader,+ quant_block_list=llm_block_names+ )+ autoround.quantize()+ try:+ import auto_gptq+ except:+ return+ autoround.save_quantized("./saved", inplace=False, safe_serialization=False)+ autoround.save_quantized(output_dir="./saved", inplace=False, format="auto_round")+++++if __name__ == "__main__":+ unittest.main()+