PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度
add pile calib, rename quant_block_list to to_quant_block_names (#322)
WeiweiZhang1d493572fb441
diff --git a/auto_round/auto_quantizer.py b/auto_round/auto_quantizer.pyindex 683d872b8..7cbcd2ea4 100644--- a/auto_round/auto_quantizer.py+++ b/auto_round/auto_quantizer.py@@ -403,9 +403,9 @@ class AutoRoundQuantizer(HfQuantizer):data_type = quantization_config.data_type if hasattr(quantization_config,"data_type") else "int" # pragma: no coversym = quantization_config.sym- quant_block_list = quantization_config.quant_block_list if hasattr(quantization_config,- "quant_block_list") else None- layer_names = get_layer_names_in_block(model, quant_block_list=quant_block_list)+ to_quant_block_names = quantization_config.to_quant_block_names if hasattr(quantization_config,+ "to_quant_block_names") else None+ layer_names = get_layer_names_in_block(model, to_quant_block_names=to_quant_block_names)extra_config = {}if hasattr(quantization_config, "extra_config"):@@ -741,3 +741,4 @@ if version.parse(transformers.__version__) < version.parse("4.38.0"):transformers.quantizers.auto.AutoHfQuantizer = AutoHfQuantizertransformers.modeling_utils.AutoHfQuantizer = AutoHfQuantizer+diff --git a/auto_round/autoround.py b/auto_round/autoround.pyindex 0bf66177d..dad317d93 100644--- a/auto_round/autoround.py+++ b/auto_round/autoround.py@@ -51,6 +51,7 @@ from .utils import (mv_module_from_gpu,unsupport_meta_device, clear_memory,compile_func,+ find_matching_blocks)from .low_cpu_mem.utils import get_layers_before_block@@ -108,10 +109,10 @@ class AutoRound(object):act_group_size (int): Group size for activation quantization. Default is None.act_sym (bool): Whether to use symmetric activation quantization. Default is None.act_dynamic (bool): Whether to use dynamic activation quantization. Default is True.- quant_block_list (list): A list whose elements are list of block's layer names to be quantized.+ to_quant_block_names (str|list): A string or list whose elements are list of+ block's layer names to be quantized.enable_norm_bias_tuning (bool): Whether to enable fast norm/layer_bias tuning- enable_torch_compile (bool): Whether to enable torch compile to optimize quant_block/layer, torch>=2.6 True-+ enable_torch_compile (bool): Whether to enable torch compile to optimize quant_block/layer, torch>=2.6 True.Returns:The quantized model."""@@ -150,7 +151,7 @@ class AutoRound(object):act_group_size: int = None,act_sym: bool = None,act_dynamic: bool = True,- quant_block_list: list = None,+ to_quant_block_names: Union[str, list] = None,enable_norm_bias_tuning: bool = False,enable_torch_compile: bool = None,**kwargs,@@ -201,7 +202,10 @@ class AutoRound(object):self.scale_dtype = convert_dtype_str2torch(scale_dtype)self.set_amp_dtype()self.cache_device = torch.device("cpu") if self.low_gpu_mem_usage else self.device- self.quant_block_list = quant_block_list+ if not hasattr(self, 'to_quant_block_names'):+ all_blocks = get_block_names(model)+ self.to_quant_block_names = find_matching_blocks(model, all_blocks, to_quant_block_names)+self.sampler = samplerself.not_use_best_mse = not_use_best_mse@@ -267,8 +271,8 @@ class AutoRound(object):The quantized model and layer configurations."""- if bool(self.quant_block_list):- all_blocks = self.quant_block_list+ if bool(self.to_quant_block_names):+ all_blocks = self.to_quant_block_nameselse:all_blocks = get_block_names(self.model)@@ -418,7 +422,7 @@ class AutoRound(object):Returns:None"""- layers_in_blocks = get_layer_names_in_block(self.model, self.supported_types, self.quant_block_list)+ layers_in_blocks = get_layer_names_in_block(self.model, self.supported_types, self.to_quant_block_names)keys = ["data_type", "bits", "group_size", "sym", "scale_dtype", "act_bits", "act_group_size", "act_sym","act_dynamic"]for n, m in self.model.named_modules():@@ -1248,7 +1252,7 @@ class AutoRound(object):"amp","nsamples","low_gpu_mem_usage",- "quant_block_list",+ "to_quant_block_names","enable_norm_bias_tuning"]if isinstance(self.dataset, str):@@ -1281,7 +1285,7 @@ class AutoRound(object):data_type=self.data_type,serialization_dict=serialization_dict,backend=backend,- quant_block_list=self.quant_block_list,+ to_quant_block_names=self.to_quant_block_names,**kwargs)return compressed_model@@ -1296,7 +1300,7 @@ class AutoRound(object):return []layer_names = []- all_layers_in_block = get_layer_names_in_block(self.model, self.supported_types, self.quant_block_list)+ all_layers_in_block = get_layer_names_in_block(self.model, self.supported_types, self.to_quant_block_names)for key in self.layer_config.keys():if key in all_layers_in_block:@@ -1419,7 +1423,8 @@ class AutoRoundOPT(AutoRound):act_group_size (int): Group size for activation quantization. Default is None.act_sym (bool): Whether to use symmetric activation quantization. Default is None.act_dynamic (bool): Whether to use dynamic activation quantization. Default is True.- quant_block_list (list): A list whose elements are list of block's layer names to be quantized.+ to_quant_block_names (str|list): A string or list whose elements are list of+ block's layer names to be quantized.enable_norm_bias_tuning (bool): Whether to enable fast norm/layer_bias tuningenable_torch_compile (bool): Whether to enable torch compile to optimize quant_block/layer function**kwargs: Additional keyword arguments.@@ -1462,7 +1467,7 @@ class AutoRoundOPT(AutoRound):act_group_size: int = None,act_sym: bool = None,act_dynamic: bool = True,- quant_block_list: list = None,+ to_quant_block_names: Union[str, list] = None,enable_norm_bias_tuning: bool = False,enable_torch_compile: bool = None,optimizer="AdamW",@@ -1501,7 +1506,7 @@ class AutoRoundOPT(AutoRound):act_group_size=act_group_size,act_sym=act_sym,act_dynamic=act_dynamic,- quant_block_list=quant_block_list,+ to_quant_block_names=to_quant_block_names,enable_norm_bias_tuning=enable_norm_bias_tuning,enable_torch_compile=enable_torch_compile,**kwargs,@@ -1589,7 +1594,7 @@ class AutoRoundAdam(AutoRoundOPT):act_group_size (int): Group size for activation quantization. Default is None.act_sym (bool): Whether to use symmetric activation quantization. Default is None.act_dynamic (bool): Whether to use dynamic activation quantization. Default is True.- quant_block_list (list): A list whose elements are list of block's layer names to be quantized.+ to_quant_block_names (str|list): A list whose elements are list of block's layer names to be quantized.enable_norm_bias_tuning (bool): Whether to enable fast norm/layer_bias tuningenable_torch_compile (bool): Whether to enable torch compile to optimize quant_block/layer functionReturns:@@ -1630,7 +1635,7 @@ class AutoRoundAdam(AutoRoundOPT):act_group_size: int = None,act_sym: bool = None,act_dynamic: bool = True,- quant_block_list: list = None,+ to_quant_block_names: Union[str, list] = None,enable_norm_bias_tuning: bool = False,enable_torch_compile: bool = None,optimizer="AdamW",@@ -1669,9 +1674,10 @@ class AutoRoundAdam(AutoRoundOPT):act_group_size=act_group_size,act_sym=act_sym,act_dynamic=act_dynamic,- quant_block_list=quant_block_list,+ to_quant_block_names=to_quant_block_names,enable_norm_bias_tuning=enable_norm_bias_tuning,enable_torch_compile=enable_torch_compile,optimizer=optimizer,**kwargs,)+diff --git a/auto_round/export/export_to_autogptq/export.py b/auto_round/export/export_to_autogptq/export.pyindex 6e8e1c90b..76ae6856d 100644--- a/auto_round/export/export_to_autogptq/export.py+++ b/auto_round/export/export_to_autogptq/export.py@@ -121,7 +121,7 @@ def save_quantized_as_autogptq(output_dir, inplace=True, backend="auto_gptq:exllmodel = kwargs["model"]supported_types = kwargs["supported_types"]safe_serialization = True if 'safe_serialization' not in kwargs.keys() else kwargs["safe_serialization"]- quant_block_list = kwargs["quant_block_list"]+ to_quant_block_names = kwargs["to_quant_block_names"]logger.info("Saving quantized model to autogptq format, this may take a while...")tokenizer = kwargs.get("tokenizer", None)processor = kwargs.get("processor", None)@@ -131,8 +131,8 @@ def save_quantized_as_autogptq(output_dir, inplace=True, backend="auto_gptq:exllprocessor.save_pretrained(output_dir)##check module quantized in block, this may have bug for mixed precision quantizationquantization_config = kwargs["serialization_dict"]- if bool(quant_block_list):- all_blocks = quant_block_list+ if bool(to_quant_block_names):+ all_blocks = to_quant_block_namesflattened_list = [item for sublist in all_blocks for item in sublist]common_prefix = os.path.commonprefix(flattened_list).rstrip('.')if common_prefix not in BLOCK_PATTERNS:@@ -220,3 +220,4 @@ def save(model: torch.nn.Module, save_dir: str, max_shard_size: str = "5GB", safif hasattr(model, "config") and hasattr(model.config, "quantization_config"):with open(os.path.join(save_dir, config_file), "w", encoding="utf-8") as f:json.dump(model.config.quantization_config, f, indent=2)+diff --git a/auto_round/export/export_to_autoround/export.py b/auto_round/export/export_to_autoround/export.pyindex f8e769e2c..1a4e33dd2 100644--- a/auto_round/export/export_to_autoround/export.py+++ b/auto_round/export/export_to_autoround/export.py@@ -188,11 +188,11 @@ def save_quantized_as_autoround(output_dir, inplace=True, backend="auto_round:exbackend = backend.replace("auto_round", "auto_gptq")model = kwargs["model"]- quant_block_list = kwargs["quant_block_list"]+ to_quant_block_names = kwargs["to_quant_block_names"]safe_serialization = True if 'safe_serialization' not in kwargs.keys() else kwargs["safe_serialization"]if not inplace:model = copy.deepcopy(model.to("cpu"))- layer_names_in_block = get_layer_names_in_block(model, quant_block_list=quant_block_list)+ layer_names_in_block = get_layer_names_in_block(model, to_quant_block_names=to_quant_block_names)layer_config = kwargs["layer_config"]quantization_config = kwargs["serialization_dict"]@@ -278,3 +278,4 @@ def save(model: nn.Module, save_dir: str, max_shard_size: str = "5GB", safe_seriwith open(os.path.join(save_dir, config_file), "w", encoding="utf-8") as f:json.dump(model.config.quantization_config, f, indent=2)+diff --git a/auto_round/mllm/autoround_mllm.py b/auto_round/mllm/autoround_mllm.pyindex a615a99bb..a3c1f8adf 100644--- a/auto_round/mllm/autoround_mllm.py+++ b/auto_round/mllm/autoround_mllm.py@@ -21,13 +21,13 @@ from ..utils import (logger,to_device,to_dtype,- get_multimodal_block_names+ get_multimodal_block_names,+ find_matching_blocks)from ..autoround import AutoRoundfrom .template import get_template, Templatefrom .mllm_dataset import get_mllm_dataloaderfrom ..low_cpu_mem.utils import get_layers_before_block-from ..special_model_handler import check_mllm_model_batchclass AutoRoundMLLM(AutoRound):@@ -58,7 +58,7 @@ class AutoRoundMLLM(AutoRound):seqlen (int): Length of the sequence.nsamples (int): Number of samples (default is 128).sampler (str): The sampling method (default is "rand").- seed (int): The random seed (default is 42).+ seed (int): The random seed (default is 42).snblocks (int): Number of blocks (default is 1).gradient_accumulate_steps (int): Number of gradient accumulation steps (default is 1).not_use_best_mse (bool): Whether to use mean squared error (default is False).@@ -70,7 +70,8 @@ class AutoRoundMLLM(AutoRound):act_group_size (int): Group size for activation quantization. Default is None.act_sym (bool): Whether to use symmetric activation quantization. Default is None.act_dynamic (bool): Whether to use dynamic activation quantization. Default is True.- quant_block_list (list): A list whose elements are list of block's layer names to be quantized.+ to_quant_block_names (str|list): A string or list whose elements are list of+ block's layer names to be quantized.enable_torch_compile (bool): Whether to enable torch compile to optimize quant_block/layer, torch>=2.6 True**kwargs: Additional keyword arguments.@@ -115,14 +116,14 @@ class AutoRoundMLLM(AutoRound):act_group_size: int = None,act_sym: bool = None,act_dynamic: bool = True,- quant_block_list: list = None,+ to_quant_block_names: Union[str, list] = None,enable_norm_bias_tuning: bool = False,truncation: bool = False,enable_torch_compile: bool = None,**kwargs,):- if quant_block_list is None:- quant_block_list = get_multimodal_block_names(model, quant_nontext_module)+ all_blocks = get_multimodal_block_names(model, quant_nontext_module)+ self.to_quant_block_names = find_matching_blocks(model, all_blocks, to_quant_block_names)self.extra_data_dir = extra_data_dirself.quant_nontext_module = quant_nontext_moduleself.image_processor = image_processor@@ -131,7 +132,6 @@ class AutoRoundMLLM(AutoRound):self.template, model=model, tokenizer=tokenizer, image_processor=image_processor)self.truncation = truncationassert dataset is not None, "dataset should not be None"- batch_size, gradient_accumulate_steps = check_mllm_model_batch(model, batch_size, gradient_accumulate_steps)super(AutoRoundMLLM, self).__init__(model=model,@@ -166,7 +166,7 @@ class AutoRoundMLLM(AutoRound):act_group_size=act_group_size,act_sym=act_sym,act_dynamic=act_dynamic,- quant_block_list=quant_block_list,+ to_quant_block_names=self.to_quant_block_names,enable_norm_bias_tuning=enable_norm_bias_tuning,enable_torch_compile=enable_torch_compile,**kwargs,@@ -186,18 +186,21 @@ class AutoRoundMLLM(AutoRound):"""if isinstance(self.dataset, str):dataset = self.dataset.replace(" ", "")- self.dataloader = get_mllm_dataloader(- template=self.template,- model=self.model,- tokenizer=self.tokenizer,- image_processor=self.image_processor,- dataset=dataset,- extra_data_dir=self.extra_data_dir,- seqlen=self.seqlen,- bs=bs,- seed=self.seed,- truncation=self.truncation,- )+ self.dataloader, self.batch_size, self.gradient_accumulate_steps = get_mllm_dataloader(+ template=self.template,+ model=self.model,+ tokenizer=self.tokenizer,+ image_processor=self.image_processor,+ dataset=dataset,+ extra_data_dir=self.extra_data_dir,+ seqlen=self.seqlen,+ bs=self.batch_size,+ seed=self.seed,+ truncation=self.truncation,+ nsamples=self.nsamples,+ gradient_accumulate_steps=self.gradient_accumulate_steps,+ quant_nontext_module=self.quant_nontext_module+ )else:self.dataloader = self.datasettotal_cnt = 0@@ -207,69 +210,72 @@ class AutoRoundMLLM(AutoRound):for n, m in embed_layers:m = m.to(self.device)- for data in tqdm(self.dataloader, desc="calib", total=nsamples-1):- if data is None:- continue- if isinstance(data, torch.Tensor):- input_ids = data.to(self.device)- data_new = input_ids- elif isinstance(data, str):- if self.tokenizer is None:- logger.error("please provide tokenizer for string input")- exit()- # data = self.template._encode(data)- data = self.template.processor.get_input(- text=data,- images=None,- max_length=self.seqlen,- squeeze=False,- )- data_new = {}- for key in data.keys():- data_new[key] = data[key].to(self.device)- input_ids = data_new["input_ids"]- elif isinstance(data, dict) and "text" in data.keys():- text = data['text']- if isinstance(text, dict):- text = [text]- input_text = self.template._encode(text)- data = self.template.processor.get_input(- text=input_text,- images=data["image"],- max_length=self.seqlen,- squeeze=False,- )- data_new = {}- for key in data.keys():- data_new[key] = to_device(data[key], self.model.device)- if key == 'images':- data_new[key] = to_dtype(data_new[key], self.model.dtype)- input_ids = data_new["input_ids"]- elif isinstance(data, tuple) or isinstance(data, list):- data_new = data- input_ids = data_new[0]- else:- data_new = {}- for key in data.keys():- data_new[key] = to_device(data[key], self.model.device)- if key == 'images':- data_new[key] = to_dtype(data_new[key], self.model.dtype)- input_ids = data_new["input_ids"]-- try:- if isinstance(data_new, torch.Tensor):- self.model(data_new)- elif isinstance(data_new, tuple) or isinstance(data_new, list):- self.model(*data_new)+ with tqdm(total=nsamples-1, desc="calib") as pbar:+ for data in self.dataloader:+ if data is None:+ continue+ if isinstance(data, torch.Tensor):+ input_ids = data.to(self.device)+ data_new = input_ids+ elif isinstance(data, str):+ if self.tokenizer is None:+ logger.error("please provide tokenizer for string input")+ exit()+ # data = self.template._encode(data)+ data = self.template.processor.get_input(+ text=data,+ images=None,+ max_length=self.seqlen,+ squeeze=False,+ )+ data_new = {}+ for key in data.keys():+ data_new[key] = data[key].to(self.device)+ input_ids = data_new["input_ids"]+ elif isinstance(data, dict) and "text" in data.keys():+ text = data['text']+ if isinstance(text, dict):+ text = [text]+ input_text = self.template._encode(text)+ data = self.template.processor.get_input(+ text=input_text,+ images=data["image"],+ max_length=self.seqlen,+ squeeze=False,+ )+ data_new = {}+ for key in data.keys():+ data_new[key] = to_device(data[key], self.model.device)+ if key == 'images':+ data_new[key] = to_dtype(data_new[key], self.model.dtype)+ input_ids = data_new["input_ids"]+ elif isinstance(data, tuple) or isinstance(data, list):+ data_new = data+ input_ids = data_new[0]else:- self.model(**data_new)- except NotImplementedError:- pass- except Exception as error:- raise error- total_cnt += input_ids.shape[0] if len(input_ids.shape) > 1 else 1- if total_cnt >= nsamples:- break+ data_new = {}+ for key in data.keys():+ data_new[key] = to_device(data[key], self.model.device)+ if key == 'images':+ data_new[key] = to_dtype(data_new[key], self.model.dtype)+ input_ids = data_new["input_ids"]++ try:+ if isinstance(data_new, torch.Tensor):+ self.model(data_new)+ elif isinstance(data_new, tuple) or isinstance(data_new, list):+ self.model(*data_new)+ else:+ self.model(**data_new)+ except NotImplementedError:+ pass+ except Exception as error:+ raise error+ step = input_ids.shape[0] if len(input_ids.shape) > 1 else 1+ total_cnt += step+ pbar.update(step)+ if total_cnt >= nsamples:+ breakif total_cnt == 0:logger.error(f"no data has been cached, please provide more data with sequence length >={self.seqlen} in the "@@ -287,3 +293,4 @@ class AutoRoundMLLM(AutoRound):for n, m in embed_layers:m = m.to("meta")# torch.cuda.empty_cache()+diff --git a/auto_round/mllm/mllm_dataset.py b/auto_round/mllm/mllm_dataset.pyindex dae71fcad..804beb57d 100644--- a/auto_round/mllm/mllm_dataset.py+++ b/auto_round/mllm/mllm_dataset.py@@ -24,6 +24,7 @@ from transformers import set_seedfrom .utils import _extract_data_dirfrom .template import Templatefrom ..utils import logger+from ..special_model_handler import check_mllm_model_batchMLLM_DATASET : Dict[str, Dataset] = {}@@ -174,12 +175,15 @@ def get_mllm_dataloader(image_processor=None,dataset="liuhaotian/llava_conv_58k",extra_data_dir=None,- seqlen=512,+ seqlen=512,bs=1,split=None,apply_template=None,truncation=False,seed=42,+ nsamples=512,+ gradient_accumulate_steps=1,+ quant_nontext_module=False):"""Generate a DataLoader for calibration using specified parameters.@@ -202,22 +206,32 @@ def get_mllm_dataloader(from .template import get_templatetemplate = get_template(template, model=model, tokenizer=tokenizer, image_processor=image_processor)- if isinstance(dataset, str):- if os.path.isfile(dataset):- dataset = MLLM_DATASET['liuhaotian/llava'](- template, model, tokenizer, dataset, extra_data_dir,- seqlen=min(seqlen, tokenizer.model_max_length), truncation=truncation)- elif "liuhaotian/llava" in dataset:- dataset = MLLM_DATASET["liuhaotian/llava"](- template, model, tokenizer, dataset, extra_data_dir,- seqlen=min(seqlen, tokenizer.model_max_length), truncation=truncation)- else:- from datasets import load_dataset- from ..calib_dataset import get_tokenizer_function- dataset = load_dataset(dataset, split=split)- tokenizer_function = get_tokenizer_function(tokenizer, seqlen, apply_template=apply_template)- dataset = dataset.map(tokenizer_function, batched=True)-+ if os.path.isfile(dataset):+ dataset = MLLM_DATASET['liuhaotian/llava'](+ template, model, tokenizer, dataset, extra_data_dir,+ seqlen=min(seqlen, tokenizer.model_max_length), truncation=truncation)+ elif "liuhaotian/llava" in dataset:+ dataset = MLLM_DATASET["liuhaotian/llava"](+ template, model, tokenizer, dataset, extra_data_dir,+ seqlen=min(seqlen, tokenizer.model_max_length), truncation=truncation)+ else:+ # try to load text calibration dataset+ from ..calib_dataset import get_dataloader+ dataloader = get_dataloader(+ tokenizer, seqlen, dataset, seed, bs, nsamples)+ if quant_nontext_module:+ logger.error(+ f"Quantitative nontext module is not supported for plain text datasets," \+ " please disable arg '--quant_nontext_module'")+ exit(-1)+ if "mllama" in model.config.model_type:+ logger.error(+ f"The llama3.2-vision model does not support the quantization of text-only calibration datasets.")+ exit(-1)+ return dataloader, bs, gradient_accumulate_steps++ bs, gradient_accumulate_steps = check_mllm_model_batch(+ model, batch_size=bs, gradient_accumulate_steps=gradient_accumulate_steps)set_seed(seed)dataloader_params = {@@ -226,4 +240,5 @@ def get_mllm_dataloader("collate_fn": dataset.template.processor.data_collator}- return DataLoader(dataset, **dataloader_params)+ return DataLoader(dataset, **dataloader_params), bs, gradient_accumulate_steps+diff --git a/auto_round/script/llm.py b/auto_round/script/llm.pyindex d153cb07b..03700d3ea 100644--- a/auto_round/script/llm.py+++ b/auto_round/script/llm.py@@ -142,6 +142,9 @@ class BasicArgumentParser(argparse.ArgumentParser):self.add_argument("--not_use_best_mse", action='store_true',help="whether to use the iter of best mes loss in the tuning phase")++ self.add_argument("--to_quant_block_names", default=None, type=str,+ help="Names of quantitative blocks, please use commas to separate them.")self.add_argument("--enable_torch_compile", default=None, type=bool,help="whether to enable torch compile")@@ -404,7 +407,7 @@ def tune(args):enable_minmax_tuning=not args.disable_minmax_tuning, act_bits=args.act_bits,low_cpu_mem_usage=low_cpu_mem_usage, data_type=args.data_type,enable_norm_bias_tuning=args.enable_norm_bias_tuning, not_use_best_mse=args.not_use_best_mse,- enable_torch_compile=args.enable_torch_compile)+ to_quant_block_names=args.to_quant_block_names, enable_torch_compile=args.enable_torch_compile)model, _ = autoround.quantize()model_name = args.model.rstrip("/")if args.low_cpu_mem_mode == 1 or args.low_cpu_mem_mode == 2:@@ -504,3 +507,4 @@ def eval(args):from lm_eval.utils import make_table # pylint: disable=E0401print(make_table(res))+diff --git a/auto_round/script/mllm.py b/auto_round/script/mllm.pyindex 5e18b9321..9afcc1d4d 100644--- a/auto_round/script/mllm.py+++ b/auto_round/script/mllm.py@@ -153,6 +153,9 @@ class BasicArgumentParser(argparse.ArgumentParser):self.add_argument("--truncation", action="store_true",help="whether to truncate sequences at the maximum length.")++ self.add_argument("--to_quant_block_names", default=None, type=str,+ help="Names of quantitative blocks, please use commas to separate them.")## ======================= VLM eval=======================self.add_argument("--tasks", type=str,@@ -341,7 +344,7 @@ def tune(args):scale_dtype=args.scale_dtype, layer_config=layer_config, template=args.template,enable_minmax_tuning=not args.disable_minmax_tuning, act_bits=args.act_bits,quant_nontext_module=args.quant_nontext_module, not_use_best_mse=args.not_use_best_mse,- enable_torch_compile=args.enable_torch_compile)+ to_quant_block_names=args.to_quant_block_names, enable_torch_compile=args.enable_torch_compile)model, _ = autoround.quantize()model.eval()@@ -385,3 +388,4 @@ def eval(args):mode=args.mode,ignore=args.ignore)+diff --git a/auto_round/special_model_handler.py b/auto_round/special_model_handler.pyindex c8d1fdcbd..36aa411a7 100644--- a/auto_round/special_model_handler.py+++ b/auto_round/special_model_handler.py@@ -15,7 +15,7 @@import torchfrom collections import UserDictshareable_keywords = ("position_ids", "cache_position", "position_embeddings")-mllms_with_limited_bs = ("llava", "qwen2-vl", "phi3_v", "mllama") # Limitations on batch_size+mllms_with_limited_bs = ("llava", "qwen2_vl", "phi3_v", "mllama") # Limitations on batch_sizeskippable_cache_keys = ("past_key_value",)def to_device(input, device=torch.device("cpu")):@@ -91,7 +91,7 @@ def check_skippable_keywords(key):return False-def check_mllm_model_batch(model, batch_size, gradient_accumulate_steps):+def check_mllm_model_batch(model, batch_size, gradient_accumulate_steps=1):"""Checks model configuration to determine if it's necessary to limit bs to avoid potential input shape mismatches."""@@ -102,3 +102,4 @@ def check_mllm_model_batch(model, batch_size, gradient_accumulate_steps):f"batch_size=1. As an alternative, set the gradient_accumulate_steps={accumulate_steps}")return 1, accumulate_stepsreturn batch_size, gradient_accumulate_steps+diff --git a/auto_round/utils.py b/auto_round/utils.pyindex 179cda0c4..3ae74f6a4 100644--- a/auto_round/utils.py+++ b/auto_round/utils.py@@ -260,7 +260,7 @@ def check_is_cpu(device):return device == torch.device("cpu") or device == "cpu"-def validate_modules(module_names):+def validate_modules(module_names, quant_vision=False, vison_blocks_names=None):"""Test a list of modules' validity.@@ -288,9 +288,50 @@ def validate_modules(module_names):if shortest_module in longest_module: # pragma: no coverraise ValueError(f"Invalid modules, at least two modules detected" \" as dependent, {shortest_module} and {longest_module}")- return True+ flag = False+ if quant_vision:+ for n,_ in module_names:+ flag = any(key not in n.lower() for key in (vison_blocks_names))+ if quant_vision and not flag:+ raise ValueError(f"Cannot find the visual block. Please reset the quant_nontext_module parameter to False, " \+ "or raise an issue at https://github.com/intel/auto-round/issues.")+ return+def find_matching_blocks(model, all_blocks, to_quant_block_names):+ """+ Find and return matching blocks in the model based on to_quant_block_names.++ Args:+ model: The model (not used in this specific function but kept for completeness).+ all_blocks: List of lists, where each inner list contains full block names in the model.+ to_quant_block_names: Comma-separated string of target block names to match.++ Returns:+ target_blocks: List of lists containing full paths of matching blocks in the model.+ """+ if not to_quant_block_names:+ return all_blocks+ to_quant_block_list = to_quant_block_names+ if isinstance(to_quant_block_names, list) or isinstance(to_quant_block_names,tuple):+ return to_quant_block_names+ if isinstance(to_quant_block_names, str):+ to_quant_block_list = [name.strip() for name in to_quant_block_names.split(",")]+ target_blocks = []+ for block_list in all_blocks:+ matched_sublist = []+ for name in to_quant_block_list:+ matches = [block for block in block_list if re.search(name, block)]+ if matches:+ matched_sublist.extend(matches)+ if matched_sublist:+ target_blocks.append(matched_sublist)+ if not target_blocks:+ raise ValueError("No block names matched. Please check the input for to_quant_block_name," \+ "or set to_quant_block_name to None to automatically match quantizable blocks.")+ return target_blocks++def get_block_names(model):"""Get the block names for transformers-like networks.@@ -324,12 +365,12 @@ def get_multimodal_block_names(model, quant_vision=False):"""block_names = []target_modules = []- Vison_blocks_tuple = ("vision", "visual",)+ vison_blocks_tuple = ("vision", "visual",)for n, m in model.named_modules():if hasattr(type(m), "__name__") and "ModuleList" in type(m).__name__:- if quant_vision or all(key not in n.lower() for key in (Vison_blocks_tuple)):+ if quant_vision or all(key not in n.lower() for key in (vison_blocks_tuple)):target_modules.append((n, m))- validate_modules(target_modules)+ validate_modules(target_modules, quant_vision, vison_blocks_tuple)for i, target_m in enumerate(target_modules):block_names.append([])for n, m in target_m[1].named_children():@@ -713,7 +754,7 @@ def check_memory_availability(device, inputs, weight, org_seqlen, org_bs):def get_layer_names_in_block(model, supported_types=[torch.nn.Linear,- transformers.modeling_utils.Conv1D], quant_block_list=None):+ transformers.modeling_utils.Conv1D], to_quant_block_names=None):"""Retrieves the names of layers within each block of the model.Returns:@@ -724,8 +765,8 @@ def get_layer_names_in_block(model, supported_types=[torch.nn.Linear,if isinstance(m, tuple(supported_types)):m.tmp_name = nlayers_in_block = []- if bool(quant_block_list):- all_blocks = quant_block_list+ if bool(to_quant_block_names):+ all_blocks = to_quant_block_nameselse:all_blocks = get_block_names(model)for block_names in all_blocks:@@ -917,3 +958,4 @@ def compile_func(fun, device, enable_torch_compile):return compile_func_on_hpu(fun) ## use auto by defaultelse:return compile_func_on_cuda_or_cpu(fun, enable_torch_compile)+diff --git a/test/test_autoopt.py b/test/test_autoopt.pyindex 08b1700bd..6dcb65543 100644--- a/test/test_autoopt.py+++ b/test/test_autoopt.py@@ -44,7 +44,7 @@ class TestAutoRound(unittest.TestCase):iters=2,seqlen=10,dataset=self.llm_dataloader,- quant_block_list=None+ to_quant_block_names=None)autoround.quantize()@@ -64,7 +64,7 @@ class TestAutoRound(unittest.TestCase):seqlen=2,batch_size=batch_size,dataset=self.llm_dataloader,- quant_block_list=llm_block_names+ to_quant_block_names=llm_block_names)adamround.quantize()@@ -72,3 +72,4 @@ class TestAutoRound(unittest.TestCase):if __name__ == "__main__":unittest.main()+diff --git a/test/test_block_names.py b/test/test_block_names.pyindex 60fc8568d..5cdbd9c93 100644--- a/test/test_block_names.py+++ b/test/test_block_names.py@@ -161,7 +161,7 @@ class TestQuantizationBlocks(unittest.TestCase):seqlen=2,batch_size=batch_size,dataset=self.llm_dataloader,- quant_block_list=llm_block_names+ to_quant_block_names=llm_block_names)autoround.quantize()try:@@ -181,7 +181,7 @@ class TestQuantizationBlocks(unittest.TestCase):print(tokenizer.decode(model.generate(**inputs, max_new_tokens=50)[0]))shutil.rmtree("./saved", ignore_errors=True)quant_config = model.config.quantization_config- assert quant_config.quant_block_list is not None+ assert quant_config.to_quant_block_names is not None@@ -191,3 +191,4 @@ if __name__ == "__main__":+diff --git a/test/test_mllm.py b/test/test_mllm.pyindex 616233f2d..ef3b62396 100644--- a/test/test_mllm.py+++ b/test/test_mllm.py@@ -67,7 +67,22 @@ class TestAutoRoundMLLM(unittest.TestCase):batch_size=1, iters=2, dataset=self.dataset, quant_nontext_module=True,seqlen=256)autoround.quantize()autoround.save_quantized("./saved/", format="auto_round", inplace=True)++ def test_quant_block_names(self):+ from auto_round.utils import get_multimodal_block_names,find_matching_blocks+ tokenizer = AutoTokenizer.from_pretrained(self.model_name)+ processor = AutoProcessor.from_pretrained(self.model_name, trust_remote_code=True)+ tokenizer.processor = processor+ model = Qwen2VLForConditionalGeneration.from_pretrained(+ self.model_name, trust_remote_code=True, device_map="auto")+ to_quant_block_names = 'visual.*12,layers.0,model.layers.*9'+ target_blocks = [['visual.blocks.12'], ['model.layers.0', 'model.layers.9', 'model.layers.19']]+ all_blocks = get_multimodal_block_names(model, quant_vision=True)+ blocks = find_matching_blocks(model, all_blocks, to_quant_block_names)+ assert target_blocks == blocks+if __name__ == "__main__":unittest.main()+