Skip to content

PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度

add pile calib, rename quant_block_list to to_quant_block_names (#322)

WeiweiZhang1d493572fb441
diff --git a/auto_round/auto_quantizer.py b/auto_round/auto_quantizer.py
index 683d872b8..7cbcd2ea4 100644
--- a/auto_round/auto_quantizer.py
+++ b/auto_round/auto_quantizer.py
@@ -403,9 +403,9 @@ class AutoRoundQuantizer(HfQuantizer):
data_type = quantization_config.data_type if hasattr(quantization_config,
"data_type") else "int" # pragma: no cover
sym = quantization_config.sym
- quant_block_list = quantization_config.quant_block_list if hasattr(quantization_config,
- "quant_block_list") else None
- layer_names = get_layer_names_in_block(model, quant_block_list=quant_block_list)
+ to_quant_block_names = quantization_config.to_quant_block_names if hasattr(quantization_config,
+ "to_quant_block_names") else None
+ layer_names = get_layer_names_in_block(model, to_quant_block_names=to_quant_block_names)
extra_config = {}
if hasattr(quantization_config, "extra_config"):
@@ -741,3 +741,4 @@ if version.parse(transformers.__version__) < version.parse("4.38.0"):
transformers.quantizers.auto.AutoHfQuantizer = AutoHfQuantizer
transformers.modeling_utils.AutoHfQuantizer = AutoHfQuantizer
+
diff --git a/auto_round/autoround.py b/auto_round/autoround.py
index 0bf66177d..dad317d93 100644
--- a/auto_round/autoround.py
+++ b/auto_round/autoround.py
@@ -51,6 +51,7 @@ from .utils import (
mv_module_from_gpu,
unsupport_meta_device, clear_memory,
compile_func,
+ find_matching_blocks
)
from .low_cpu_mem.utils import get_layers_before_block
@@ -108,10 +109,10 @@ class AutoRound(object):
act_group_size (int): Group size for activation quantization. Default is None.
act_sym (bool): Whether to use symmetric activation quantization. Default is None.
act_dynamic (bool): Whether to use dynamic activation quantization. Default is True.
- quant_block_list (list): A list whose elements are list of block's layer names to be quantized.
+ to_quant_block_names (str|list): A string or list whose elements are list of
+ block's layer names to be quantized.
enable_norm_bias_tuning (bool): Whether to enable fast norm/layer_bias tuning
- enable_torch_compile (bool): Whether to enable torch compile to optimize quant_block/layer, torch>=2.6 True
-
+ enable_torch_compile (bool): Whether to enable torch compile to optimize quant_block/layer, torch>=2.6 True.
Returns:
The quantized model.
"""
@@ -150,7 +151,7 @@ class AutoRound(object):
act_group_size: int = None,
act_sym: bool = None,
act_dynamic: bool = True,
- quant_block_list: list = None,
+ to_quant_block_names: Union[str, list] = None,
enable_norm_bias_tuning: bool = False,
enable_torch_compile: bool = None,
**kwargs,
@@ -201,7 +202,10 @@ class AutoRound(object):
self.scale_dtype = convert_dtype_str2torch(scale_dtype)
self.set_amp_dtype()
self.cache_device = torch.device("cpu") if self.low_gpu_mem_usage else self.device
- self.quant_block_list = quant_block_list
+ if not hasattr(self, 'to_quant_block_names'):
+ all_blocks = get_block_names(model)
+ self.to_quant_block_names = find_matching_blocks(model, all_blocks, to_quant_block_names)
+
self.sampler = sampler
self.not_use_best_mse = not_use_best_mse
@@ -267,8 +271,8 @@ class AutoRound(object):
The quantized model and layer configurations.
"""
- if bool(self.quant_block_list):
- all_blocks = self.quant_block_list
+ if bool(self.to_quant_block_names):
+ all_blocks = self.to_quant_block_names
else:
all_blocks = get_block_names(self.model)
@@ -418,7 +422,7 @@ class AutoRound(object):
Returns:
None
"""
- layers_in_blocks = get_layer_names_in_block(self.model, self.supported_types, self.quant_block_list)
+ layers_in_blocks = get_layer_names_in_block(self.model, self.supported_types, self.to_quant_block_names)
keys = ["data_type", "bits", "group_size", "sym", "scale_dtype", "act_bits", "act_group_size", "act_sym",
"act_dynamic"]
for n, m in self.model.named_modules():
@@ -1248,7 +1252,7 @@ class AutoRound(object):
"amp",
"nsamples",
"low_gpu_mem_usage",
- "quant_block_list",
+ "to_quant_block_names",
"enable_norm_bias_tuning"
]
if isinstance(self.dataset, str):
@@ -1281,7 +1285,7 @@ class AutoRound(object):
data_type=self.data_type,
serialization_dict=serialization_dict,
backend=backend,
- quant_block_list=self.quant_block_list,
+ to_quant_block_names=self.to_quant_block_names,
**kwargs
)
return compressed_model
@@ -1296,7 +1300,7 @@ class AutoRound(object):
return []
layer_names = []
- all_layers_in_block = get_layer_names_in_block(self.model, self.supported_types, self.quant_block_list)
+ all_layers_in_block = get_layer_names_in_block(self.model, self.supported_types, self.to_quant_block_names)
for key in self.layer_config.keys():
if key in all_layers_in_block:
@@ -1419,7 +1423,8 @@ class AutoRoundOPT(AutoRound):
act_group_size (int): Group size for activation quantization. Default is None.
act_sym (bool): Whether to use symmetric activation quantization. Default is None.
act_dynamic (bool): Whether to use dynamic activation quantization. Default is True.
- quant_block_list (list): A list whose elements are list of block's layer names to be quantized.
+ to_quant_block_names (str|list): A string or list whose elements are list of
+ block's layer names to be quantized.
enable_norm_bias_tuning (bool): Whether to enable fast norm/layer_bias tuning
enable_torch_compile (bool): Whether to enable torch compile to optimize quant_block/layer function
**kwargs: Additional keyword arguments.
@@ -1462,7 +1467,7 @@ class AutoRoundOPT(AutoRound):
act_group_size: int = None,
act_sym: bool = None,
act_dynamic: bool = True,
- quant_block_list: list = None,
+ to_quant_block_names: Union[str, list] = None,
enable_norm_bias_tuning: bool = False,
enable_torch_compile: bool = None,
optimizer="AdamW",
@@ -1501,7 +1506,7 @@ class AutoRoundOPT(AutoRound):
act_group_size=act_group_size,
act_sym=act_sym,
act_dynamic=act_dynamic,
- quant_block_list=quant_block_list,
+ to_quant_block_names=to_quant_block_names,
enable_norm_bias_tuning=enable_norm_bias_tuning,
enable_torch_compile=enable_torch_compile,
**kwargs,
@@ -1589,7 +1594,7 @@ class AutoRoundAdam(AutoRoundOPT):
act_group_size (int): Group size for activation quantization. Default is None.
act_sym (bool): Whether to use symmetric activation quantization. Default is None.
act_dynamic (bool): Whether to use dynamic activation quantization. Default is True.
- quant_block_list (list): A list whose elements are list of block's layer names to be quantized.
+ to_quant_block_names (str|list): A list whose elements are list of block's layer names to be quantized.
enable_norm_bias_tuning (bool): Whether to enable fast norm/layer_bias tuning
enable_torch_compile (bool): Whether to enable torch compile to optimize quant_block/layer function
Returns:
@@ -1630,7 +1635,7 @@ class AutoRoundAdam(AutoRoundOPT):
act_group_size: int = None,
act_sym: bool = None,
act_dynamic: bool = True,
- quant_block_list: list = None,
+ to_quant_block_names: Union[str, list] = None,
enable_norm_bias_tuning: bool = False,
enable_torch_compile: bool = None,
optimizer="AdamW",
@@ -1669,9 +1674,10 @@ class AutoRoundAdam(AutoRoundOPT):
act_group_size=act_group_size,
act_sym=act_sym,
act_dynamic=act_dynamic,
- quant_block_list=quant_block_list,
+ to_quant_block_names=to_quant_block_names,
enable_norm_bias_tuning=enable_norm_bias_tuning,
enable_torch_compile=enable_torch_compile,
optimizer=optimizer,
**kwargs,
)
+
diff --git a/auto_round/export/export_to_autogptq/export.py b/auto_round/export/export_to_autogptq/export.py
index 6e8e1c90b..76ae6856d 100644
--- a/auto_round/export/export_to_autogptq/export.py
+++ b/auto_round/export/export_to_autogptq/export.py
@@ -121,7 +121,7 @@ def save_quantized_as_autogptq(output_dir, inplace=True, backend="auto_gptq:exll
model = kwargs["model"]
supported_types = kwargs["supported_types"]
safe_serialization = True if 'safe_serialization' not in kwargs.keys() else kwargs["safe_serialization"]
- quant_block_list = kwargs["quant_block_list"]
+ to_quant_block_names = kwargs["to_quant_block_names"]
logger.info("Saving quantized model to autogptq format, this may take a while...")
tokenizer = kwargs.get("tokenizer", None)
processor = kwargs.get("processor", None)
@@ -131,8 +131,8 @@ def save_quantized_as_autogptq(output_dir, inplace=True, backend="auto_gptq:exll
processor.save_pretrained(output_dir)
##check module quantized in block, this may have bug for mixed precision quantization
quantization_config = kwargs["serialization_dict"]
- if bool(quant_block_list):
- all_blocks = quant_block_list
+ if bool(to_quant_block_names):
+ all_blocks = to_quant_block_names
flattened_list = [item for sublist in all_blocks for item in sublist]
common_prefix = os.path.commonprefix(flattened_list).rstrip('.')
if common_prefix not in BLOCK_PATTERNS:
@@ -220,3 +220,4 @@ def save(model: torch.nn.Module, save_dir: str, max_shard_size: str = "5GB", saf
if hasattr(model, "config") and hasattr(model.config, "quantization_config"):
with open(os.path.join(save_dir, config_file), "w", encoding="utf-8") as f:
json.dump(model.config.quantization_config, f, indent=2)
+
diff --git a/auto_round/export/export_to_autoround/export.py b/auto_round/export/export_to_autoround/export.py
index f8e769e2c..1a4e33dd2 100644
--- a/auto_round/export/export_to_autoround/export.py
+++ b/auto_round/export/export_to_autoround/export.py
@@ -188,11 +188,11 @@ def save_quantized_as_autoround(output_dir, inplace=True, backend="auto_round:ex
backend = backend.replace("auto_round", "auto_gptq")
model = kwargs["model"]
- quant_block_list = kwargs["quant_block_list"]
+ to_quant_block_names = kwargs["to_quant_block_names"]
safe_serialization = True if 'safe_serialization' not in kwargs.keys() else kwargs["safe_serialization"]
if not inplace:
model = copy.deepcopy(model.to("cpu"))
- layer_names_in_block = get_layer_names_in_block(model, quant_block_list=quant_block_list)
+ layer_names_in_block = get_layer_names_in_block(model, to_quant_block_names=to_quant_block_names)
layer_config = kwargs["layer_config"]
quantization_config = kwargs["serialization_dict"]
@@ -278,3 +278,4 @@ def save(model: nn.Module, save_dir: str, max_shard_size: str = "5GB", safe_seri
with open(os.path.join(save_dir, config_file), "w", encoding="utf-8") as f:
json.dump(model.config.quantization_config, f, indent=2)
+
diff --git a/auto_round/mllm/autoround_mllm.py b/auto_round/mllm/autoround_mllm.py
index a615a99bb..a3c1f8adf 100644
--- a/auto_round/mllm/autoround_mllm.py
+++ b/auto_round/mllm/autoround_mllm.py
@@ -21,13 +21,13 @@ from ..utils import (
logger,
to_device,
to_dtype,
- get_multimodal_block_names
+ get_multimodal_block_names,
+ find_matching_blocks
)
from ..autoround import AutoRound
from .template import get_template, Template
from .mllm_dataset import get_mllm_dataloader
from ..low_cpu_mem.utils import get_layers_before_block
-from ..special_model_handler import check_mllm_model_batch
class AutoRoundMLLM(AutoRound):
@@ -58,7 +58,7 @@ class AutoRoundMLLM(AutoRound):
seqlen (int): Length of the sequence.
nsamples (int): Number of samples (default is 128).
sampler (str): The sampling method (default is "rand").
- seed (int): The random seed (default is 42).
+ seed (int): The random seed (default is 42).s
nblocks (int): Number of blocks (default is 1).
gradient_accumulate_steps (int): Number of gradient accumulation steps (default is 1).
not_use_best_mse (bool): Whether to use mean squared error (default is False).
@@ -70,7 +70,8 @@ class AutoRoundMLLM(AutoRound):
act_group_size (int): Group size for activation quantization. Default is None.
act_sym (bool): Whether to use symmetric activation quantization. Default is None.
act_dynamic (bool): Whether to use dynamic activation quantization. Default is True.
- quant_block_list (list): A list whose elements are list of block's layer names to be quantized.
+ to_quant_block_names (str|list): A string or list whose elements are list of
+ block's layer names to be quantized.
enable_torch_compile (bool): Whether to enable torch compile to optimize quant_block/layer, torch>=2.6 True
**kwargs: Additional keyword arguments.
@@ -115,14 +116,14 @@ class AutoRoundMLLM(AutoRound):
act_group_size: int = None,
act_sym: bool = None,
act_dynamic: bool = True,
- quant_block_list: list = None,
+ to_quant_block_names: Union[str, list] = None,
enable_norm_bias_tuning: bool = False,
truncation: bool = False,
enable_torch_compile: bool = None,
**kwargs,
):
- if quant_block_list is None:
- quant_block_list = get_multimodal_block_names(model, quant_nontext_module)
+ all_blocks = get_multimodal_block_names(model, quant_nontext_module)
+ self.to_quant_block_names = find_matching_blocks(model, all_blocks, to_quant_block_names)
self.extra_data_dir = extra_data_dir
self.quant_nontext_module = quant_nontext_module
self.image_processor = image_processor
@@ -131,7 +132,6 @@ class AutoRoundMLLM(AutoRound):
self.template, model=model, tokenizer=tokenizer, image_processor=image_processor)
self.truncation = truncation
assert dataset is not None, "dataset should not be None"
- batch_size, gradient_accumulate_steps = check_mllm_model_batch(model, batch_size, gradient_accumulate_steps)
super(AutoRoundMLLM, self).__init__(
model=model,
@@ -166,7 +166,7 @@ class AutoRoundMLLM(AutoRound):
act_group_size=act_group_size,
act_sym=act_sym,
act_dynamic=act_dynamic,
- quant_block_list=quant_block_list,
+ to_quant_block_names=self.to_quant_block_names,
enable_norm_bias_tuning=enable_norm_bias_tuning,
enable_torch_compile=enable_torch_compile,
**kwargs,
@@ -186,18 +186,21 @@ class AutoRoundMLLM(AutoRound):
"""
if isinstance(self.dataset, str):
dataset = self.dataset.replace(" ", "")
- self.dataloader = get_mllm_dataloader(
- template=self.template,
- model=self.model,
- tokenizer=self.tokenizer,
- image_processor=self.image_processor,
- dataset=dataset,
- extra_data_dir=self.extra_data_dir,
- seqlen=self.seqlen,
- bs=bs,
- seed=self.seed,
- truncation=self.truncation,
- )
+ self.dataloader, self.batch_size, self.gradient_accumulate_steps = get_mllm_dataloader(
+ template=self.template,
+ model=self.model,
+ tokenizer=self.tokenizer,
+ image_processor=self.image_processor,
+ dataset=dataset,
+ extra_data_dir=self.extra_data_dir,
+ seqlen=self.seqlen,
+ bs=self.batch_size,
+ seed=self.seed,
+ truncation=self.truncation,
+ nsamples=self.nsamples,
+ gradient_accumulate_steps=self.gradient_accumulate_steps,
+ quant_nontext_module=self.quant_nontext_module
+ )
else:
self.dataloader = self.dataset
total_cnt = 0
@@ -207,69 +210,72 @@ class AutoRoundMLLM(AutoRound):
for n, m in embed_layers:
m = m.to(self.device)
- for data in tqdm(self.dataloader, desc="calib", total=nsamples-1):
- if data is None:
- continue
- if isinstance(data, torch.Tensor):
- input_ids = data.to(self.device)
- data_new = input_ids
- elif isinstance(data, str):
- if self.tokenizer is None:
- logger.error("please provide tokenizer for string input")
- exit()
- # data = self.template._encode(data)
- data = self.template.processor.get_input(
- text=data,
- images=None,
- max_length=self.seqlen,
- squeeze=False,
- )
- data_new = {}
- for key in data.keys():
- data_new[key] = data[key].to(self.device)
- input_ids = data_new["input_ids"]
- elif isinstance(data, dict) and "text" in data.keys():
- text = data['text']
- if isinstance(text, dict):
- text = [text]
- input_text = self.template._encode(text)
- data = self.template.processor.get_input(
- text=input_text,
- images=data["image"],
- max_length=self.seqlen,
- squeeze=False,
- )
- data_new = {}
- for key in data.keys():
- data_new[key] = to_device(data[key], self.model.device)
- if key == 'images':
- data_new[key] = to_dtype(data_new[key], self.model.dtype)
- input_ids = data_new["input_ids"]
- elif isinstance(data, tuple) or isinstance(data, list):
- data_new = data
- input_ids = data_new[0]
- else:
- data_new = {}
- for key in data.keys():
- data_new[key] = to_device(data[key], self.model.device)
- if key == 'images':
- data_new[key] = to_dtype(data_new[key], self.model.dtype)
- input_ids = data_new["input_ids"]
-
- try:
- if isinstance(data_new, torch.Tensor):
- self.model(data_new)
- elif isinstance(data_new, tuple) or isinstance(data_new, list):
- self.model(*data_new)
+ with tqdm(total=nsamples-1, desc="calib") as pbar:
+ for data in self.dataloader:
+ if data is None:
+ continue
+ if isinstance(data, torch.Tensor):
+ input_ids = data.to(self.device)
+ data_new = input_ids
+ elif isinstance(data, str):
+ if self.tokenizer is None:
+ logger.error("please provide tokenizer for string input")
+ exit()
+ # data = self.template._encode(data)
+ data = self.template.processor.get_input(
+ text=data,
+ images=None,
+ max_length=self.seqlen,
+ squeeze=False,
+ )
+ data_new = {}
+ for key in data.keys():
+ data_new[key] = data[key].to(self.device)
+ input_ids = data_new["input_ids"]
+ elif isinstance(data, dict) and "text" in data.keys():
+ text = data['text']
+ if isinstance(text, dict):
+ text = [text]
+ input_text = self.template._encode(text)
+ data = self.template.processor.get_input(
+ text=input_text,
+ images=data["image"],
+ max_length=self.seqlen,
+ squeeze=False,
+ )
+ data_new = {}
+ for key in data.keys():
+ data_new[key] = to_device(data[key], self.model.device)
+ if key == 'images':
+ data_new[key] = to_dtype(data_new[key], self.model.dtype)
+ input_ids = data_new["input_ids"]
+ elif isinstance(data, tuple) or isinstance(data, list):
+ data_new = data
+ input_ids = data_new[0]
else:
- self.model(**data_new)
- except NotImplementedError:
- pass
- except Exception as error:
- raise error
- total_cnt += input_ids.shape[0] if len(input_ids.shape) > 1 else 1
- if total_cnt >= nsamples:
- break
+ data_new = {}
+ for key in data.keys():
+ data_new[key] = to_device(data[key], self.model.device)
+ if key == 'images':
+ data_new[key] = to_dtype(data_new[key], self.model.dtype)
+ input_ids = data_new["input_ids"]
+
+ try:
+ if isinstance(data_new, torch.Tensor):
+ self.model(data_new)
+ elif isinstance(data_new, tuple) or isinstance(data_new, list):
+ self.model(*data_new)
+ else:
+ self.model(**data_new)
+ except NotImplementedError:
+ pass
+ except Exception as error:
+ raise error
+ step = input_ids.shape[0] if len(input_ids.shape) > 1 else 1
+ total_cnt += step
+ pbar.update(step)
+ if total_cnt >= nsamples:
+ break
if total_cnt == 0:
logger.error(
f"no data has been cached, please provide more data with sequence length >={self.seqlen} in the "
@@ -287,3 +293,4 @@ class AutoRoundMLLM(AutoRound):
for n, m in embed_layers:
m = m.to("meta")
# torch.cuda.empty_cache()
+
diff --git a/auto_round/mllm/mllm_dataset.py b/auto_round/mllm/mllm_dataset.py
index dae71fcad..804beb57d 100644
--- a/auto_round/mllm/mllm_dataset.py
+++ b/auto_round/mllm/mllm_dataset.py
@@ -24,6 +24,7 @@ from transformers import set_seed
from .utils import _extract_data_dir
from .template import Template
from ..utils import logger
+from ..special_model_handler import check_mllm_model_batch
MLLM_DATASET : Dict[str, Dataset] = {}
@@ -174,12 +175,15 @@ def get_mllm_dataloader(
image_processor=None,
dataset="liuhaotian/llava_conv_58k",
extra_data_dir=None,
- seqlen=512,
+ seqlen=512,
bs=1,
split=None,
apply_template=None,
truncation=False,
seed=42,
+ nsamples=512,
+ gradient_accumulate_steps=1,
+ quant_nontext_module=False
):
"""Generate a DataLoader for calibration using specified parameters.
@@ -202,22 +206,32 @@ def get_mllm_dataloader(
from .template import get_template
template = get_template(template, model=model, tokenizer=tokenizer, image_processor=image_processor)
- if isinstance(dataset, str):
- if os.path.isfile(dataset):
- dataset = MLLM_DATASET['liuhaotian/llava'](
- template, model, tokenizer, dataset, extra_data_dir,
- seqlen=min(seqlen, tokenizer.model_max_length), truncation=truncation)
- elif "liuhaotian/llava" in dataset:
- dataset = MLLM_DATASET["liuhaotian/llava"](
- template, model, tokenizer, dataset, extra_data_dir,
- seqlen=min(seqlen, tokenizer.model_max_length), truncation=truncation)
- else:
- from datasets import load_dataset
- from ..calib_dataset import get_tokenizer_function
- dataset = load_dataset(dataset, split=split)
- tokenizer_function = get_tokenizer_function(tokenizer, seqlen, apply_template=apply_template)
- dataset = dataset.map(tokenizer_function, batched=True)
-
+ if os.path.isfile(dataset):
+ dataset = MLLM_DATASET['liuhaotian/llava'](
+ template, model, tokenizer, dataset, extra_data_dir,
+ seqlen=min(seqlen, tokenizer.model_max_length), truncation=truncation)
+ elif "liuhaotian/llava" in dataset:
+ dataset = MLLM_DATASET["liuhaotian/llava"](
+ template, model, tokenizer, dataset, extra_data_dir,
+ seqlen=min(seqlen, tokenizer.model_max_length), truncation=truncation)
+ else:
+ # try to load text calibration dataset
+ from ..calib_dataset import get_dataloader
+ dataloader = get_dataloader(
+ tokenizer, seqlen, dataset, seed, bs, nsamples)
+ if quant_nontext_module:
+ logger.error(
+ f"Quantitative nontext module is not supported for plain text datasets," \
+ " please disable arg '--quant_nontext_module'")
+ exit(-1)
+ if "mllama" in model.config.model_type:
+ logger.error(
+ f"The llama3.2-vision model does not support the quantization of text-only calibration datasets.")
+ exit(-1)
+ return dataloader, bs, gradient_accumulate_steps
+
+ bs, gradient_accumulate_steps = check_mllm_model_batch(
+ model, batch_size=bs, gradient_accumulate_steps=gradient_accumulate_steps)
set_seed(seed)
dataloader_params = {
@@ -226,4 +240,5 @@ def get_mllm_dataloader(
"collate_fn": dataset.template.processor.data_collator
}
- return DataLoader(dataset, **dataloader_params)
+ return DataLoader(dataset, **dataloader_params), bs, gradient_accumulate_steps
+
diff --git a/auto_round/script/llm.py b/auto_round/script/llm.py
index d153cb07b..03700d3ea 100644
--- a/auto_round/script/llm.py
+++ b/auto_round/script/llm.py
@@ -142,6 +142,9 @@ class BasicArgumentParser(argparse.ArgumentParser):
self.add_argument("--not_use_best_mse", action='store_true',
help="whether to use the iter of best mes loss in the tuning phase")
+
+ self.add_argument("--to_quant_block_names", default=None, type=str,
+ help="Names of quantitative blocks, please use commas to separate them.")
self.add_argument("--enable_torch_compile", default=None, type=bool,
help="whether to enable torch compile")
@@ -404,7 +407,7 @@ def tune(args):
enable_minmax_tuning=not args.disable_minmax_tuning, act_bits=args.act_bits,
low_cpu_mem_usage=low_cpu_mem_usage, data_type=args.data_type,
enable_norm_bias_tuning=args.enable_norm_bias_tuning, not_use_best_mse=args.not_use_best_mse,
- enable_torch_compile=args.enable_torch_compile)
+ to_quant_block_names=args.to_quant_block_names, enable_torch_compile=args.enable_torch_compile)
model, _ = autoround.quantize()
model_name = args.model.rstrip("/")
if args.low_cpu_mem_mode == 1 or args.low_cpu_mem_mode == 2:
@@ -504,3 +507,4 @@ def eval(args):
from lm_eval.utils import make_table # pylint: disable=E0401
print(make_table(res))
+
diff --git a/auto_round/script/mllm.py b/auto_round/script/mllm.py
index 5e18b9321..9afcc1d4d 100644
--- a/auto_round/script/mllm.py
+++ b/auto_round/script/mllm.py
@@ -153,6 +153,9 @@ class BasicArgumentParser(argparse.ArgumentParser):
self.add_argument("--truncation", action="store_true",
help="whether to truncate sequences at the maximum length.")
+
+ self.add_argument("--to_quant_block_names", default=None, type=str,
+ help="Names of quantitative blocks, please use commas to separate them.")
## ======================= VLM eval=======================
self.add_argument("--tasks", type=str,
@@ -341,7 +344,7 @@ def tune(args):
scale_dtype=args.scale_dtype, layer_config=layer_config, template=args.template,
enable_minmax_tuning=not args.disable_minmax_tuning, act_bits=args.act_bits,
quant_nontext_module=args.quant_nontext_module, not_use_best_mse=args.not_use_best_mse,
- enable_torch_compile=args.enable_torch_compile)
+ to_quant_block_names=args.to_quant_block_names, enable_torch_compile=args.enable_torch_compile)
model, _ = autoround.quantize()
model.eval()
@@ -385,3 +388,4 @@ def eval(args):
mode=args.mode,
ignore=args.ignore
)
+
diff --git a/auto_round/special_model_handler.py b/auto_round/special_model_handler.py
index c8d1fdcbd..36aa411a7 100644
--- a/auto_round/special_model_handler.py
+++ b/auto_round/special_model_handler.py
@@ -15,7 +15,7 @@
import torch
from collections import UserDict
shareable_keywords = ("position_ids", "cache_position", "position_embeddings")
-mllms_with_limited_bs = ("llava", "qwen2-vl", "phi3_v", "mllama") # Limitations on batch_size
+mllms_with_limited_bs = ("llava", "qwen2_vl", "phi3_v", "mllama") # Limitations on batch_size
skippable_cache_keys = ("past_key_value",)
def to_device(input, device=torch.device("cpu")):
@@ -91,7 +91,7 @@ def check_skippable_keywords(key):
return False
-def check_mllm_model_batch(model, batch_size, gradient_accumulate_steps):
+def check_mllm_model_batch(model, batch_size, gradient_accumulate_steps=1):
"""
Checks model configuration to determine if it's necessary to limit bs to avoid potential input shape mismatches.
"""
@@ -102,3 +102,4 @@ def check_mllm_model_batch(model, batch_size, gradient_accumulate_steps):
f"batch_size=1. As an alternative, set the gradient_accumulate_steps={accumulate_steps}")
return 1, accumulate_steps
return batch_size, gradient_accumulate_steps
+
diff --git a/auto_round/utils.py b/auto_round/utils.py
index 179cda0c4..3ae74f6a4 100644
--- a/auto_round/utils.py
+++ b/auto_round/utils.py
@@ -260,7 +260,7 @@ def check_is_cpu(device):
return device == torch.device("cpu") or device == "cpu"
-def validate_modules(module_names):
+def validate_modules(module_names, quant_vision=False, vison_blocks_names=None):
"""
Test a list of modules' validity.
@@ -288,9 +288,50 @@ def validate_modules(module_names):
if shortest_module in longest_module: # pragma: no cover
raise ValueError(f"Invalid modules, at least two modules detected" \
" as dependent, {shortest_module} and {longest_module}")
- return True
+ flag = False
+ if quant_vision:
+ for n,_ in module_names:
+ flag = any(key not in n.lower() for key in (vison_blocks_names))
+ if quant_vision and not flag:
+ raise ValueError(f"Cannot find the visual block. Please reset the quant_nontext_module parameter to False, " \
+ "or raise an issue at https://github.com/intel/auto-round/issues.")
+ return
+def find_matching_blocks(model, all_blocks, to_quant_block_names):
+ """
+ Find and return matching blocks in the model based on to_quant_block_names.
+
+ Args:
+ model: The model (not used in this specific function but kept for completeness).
+ all_blocks: List of lists, where each inner list contains full block names in the model.
+ to_quant_block_names: Comma-separated string of target block names to match.
+
+ Returns:
+ target_blocks: List of lists containing full paths of matching blocks in the model.
+ """
+ if not to_quant_block_names:
+ return all_blocks
+ to_quant_block_list = to_quant_block_names
+ if isinstance(to_quant_block_names, list) or isinstance(to_quant_block_names,tuple):
+ return to_quant_block_names
+ if isinstance(to_quant_block_names, str):
+ to_quant_block_list = [name.strip() for name in to_quant_block_names.split(",")]
+ target_blocks = []
+ for block_list in all_blocks:
+ matched_sublist = []
+ for name in to_quant_block_list:
+ matches = [block for block in block_list if re.search(name, block)]
+ if matches:
+ matched_sublist.extend(matches)
+ if matched_sublist:
+ target_blocks.append(matched_sublist)
+ if not target_blocks:
+ raise ValueError("No block names matched. Please check the input for to_quant_block_name," \
+ "or set to_quant_block_name to None to automatically match quantizable blocks.")
+ return target_blocks
+
+
def get_block_names(model):
"""Get the block names for transformers-like networks.
@@ -324,12 +365,12 @@ def get_multimodal_block_names(model, quant_vision=False):
"""
block_names = []
target_modules = []
- Vison_blocks_tuple = ("vision", "visual",)
+ vison_blocks_tuple = ("vision", "visual",)
for n, m in model.named_modules():
if hasattr(type(m), "__name__") and "ModuleList" in type(m).__name__:
- if quant_vision or all(key not in n.lower() for key in (Vison_blocks_tuple)):
+ if quant_vision or all(key not in n.lower() for key in (vison_blocks_tuple)):
target_modules.append((n, m))
- validate_modules(target_modules)
+ validate_modules(target_modules, quant_vision, vison_blocks_tuple)
for i, target_m in enumerate(target_modules):
block_names.append([])
for n, m in target_m[1].named_children():
@@ -713,7 +754,7 @@ def check_memory_availability(device, inputs, weight, org_seqlen, org_bs):
def get_layer_names_in_block(model, supported_types=[torch.nn.Linear,
- transformers.modeling_utils.Conv1D], quant_block_list=None):
+ transformers.modeling_utils.Conv1D], to_quant_block_names=None):
"""Retrieves the names of layers within each block of the model.
Returns:
@@ -724,8 +765,8 @@ def get_layer_names_in_block(model, supported_types=[torch.nn.Linear,
if isinstance(m, tuple(supported_types)):
m.tmp_name = n
layers_in_block = []
- if bool(quant_block_list):
- all_blocks = quant_block_list
+ if bool(to_quant_block_names):
+ all_blocks = to_quant_block_names
else:
all_blocks = get_block_names(model)
for block_names in all_blocks:
@@ -917,3 +958,4 @@ def compile_func(fun, device, enable_torch_compile):
return compile_func_on_hpu(fun) ## use auto by default
else:
return compile_func_on_cuda_or_cpu(fun, enable_torch_compile)
+
diff --git a/test/test_autoopt.py b/test/test_autoopt.py
index 08b1700bd..6dcb65543 100644
--- a/test/test_autoopt.py
+++ b/test/test_autoopt.py
@@ -44,7 +44,7 @@ class TestAutoRound(unittest.TestCase):
iters=2,
seqlen=10,
dataset=self.llm_dataloader,
- quant_block_list=None
+ to_quant_block_names=None
)
autoround.quantize()
@@ -64,7 +64,7 @@ class TestAutoRound(unittest.TestCase):
seqlen=2,
batch_size=batch_size,
dataset=self.llm_dataloader,
- quant_block_list=llm_block_names
+ to_quant_block_names=llm_block_names
)
adamround.quantize()
@@ -72,3 +72,4 @@ class TestAutoRound(unittest.TestCase):
if __name__ == "__main__":
unittest.main()
+
diff --git a/test/test_block_names.py b/test/test_block_names.py
index 60fc8568d..5cdbd9c93 100644
--- a/test/test_block_names.py
+++ b/test/test_block_names.py
@@ -161,7 +161,7 @@ class TestQuantizationBlocks(unittest.TestCase):
seqlen=2,
batch_size=batch_size,
dataset=self.llm_dataloader,
- quant_block_list=llm_block_names
+ to_quant_block_names=llm_block_names
)
autoround.quantize()
try:
@@ -181,7 +181,7 @@ class TestQuantizationBlocks(unittest.TestCase):
print(tokenizer.decode(model.generate(**inputs, max_new_tokens=50)[0]))
shutil.rmtree("./saved", ignore_errors=True)
quant_config = model.config.quantization_config
- assert quant_config.quant_block_list is not None
+ assert quant_config.to_quant_block_names is not None
@@ -191,3 +191,4 @@ if __name__ == "__main__":
+
diff --git a/test/test_mllm.py b/test/test_mllm.py
index 616233f2d..ef3b62396 100644
--- a/test/test_mllm.py
+++ b/test/test_mllm.py
@@ -67,7 +67,22 @@ class TestAutoRoundMLLM(unittest.TestCase):
batch_size=1, iters=2, dataset=self.dataset, quant_nontext_module=True,seqlen=256)
autoround.quantize()
autoround.save_quantized("./saved/", format="auto_round", inplace=True)
+
+ def test_quant_block_names(self):
+ from auto_round.utils import get_multimodal_block_names,find_matching_blocks
+ tokenizer = AutoTokenizer.from_pretrained(self.model_name)
+ processor = AutoProcessor.from_pretrained(self.model_name, trust_remote_code=True)
+ tokenizer.processor = processor
+ model = Qwen2VLForConditionalGeneration.from_pretrained(
+ self.model_name, trust_remote_code=True, device_map="auto")
+ to_quant_block_names = 'visual.*12,layers.0,model.layers.*9'
+ target_blocks = [['visual.blocks.12'], ['model.layers.0', 'model.layers.9', 'model.layers.19']]
+ all_blocks = get_multimodal_block_names(model, quant_vision=True)
+ blocks = find_matching_blocks(model, all_blocks, to_quant_block_names)
+ assert target_blocks == blocks
+
if __name__ == "__main__":
unittest.main()
+