Skip to content

PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度

Fix multimodal and moe issue (#191)

WeiweiZhang1ca59d36a9407
diff --git a/auto_round/autoround.py b/auto_round/autoround.py
index 674b758bc..5b4b30a9e 100644
--- a/auto_round/autoround.py
+++ b/auto_round/autoround.py
@@ -68,8 +68,8 @@ class AutoRound(object):
'sym': False
'act_data_type': None,
'act_bits': 32,
- 'group_size': None,
- 'sym': None,
+ 'act_group_size': None,
+ 'act_sym': None,
}
...
@@ -99,12 +99,11 @@ class AutoRound(object):
data_type (str): The data type to be used (default is "int").
scale_dtype (str): The data type of quantization scale to be used (default is "float16"), different kernels
have different choices.
- multimodal(bool): Enable multimodal model quantization, (default is "False").
act_bits (int): Number of bits for activation quantization. Default is 32.
act_group_size (int): Group size for activation quantization. Default is None.
act_sym (bool): Whether to use symmetric activation quantization. Default is None.
act_dynamic (bool): Whether to use dynamic activation quantization. Default is True.
-
+ quant_block_list (list): A list whose elements are list of block's layer names to be quantized.
Returns:
The quantized model.
"""
@@ -140,11 +139,11 @@ class AutoRound(object):
dynamic_max_gap: int = -1,
data_type: str = "int",
scale_dtype: str = "fp16",
- multimodal:bool = False,
act_bits: int = 32,
act_group_size: int = None,
act_sym: bool = None,
act_dynamic: bool = True,
+ quant_block_list: list = None,
**kwargs,
):
self.quantized = False
@@ -176,7 +175,7 @@ class AutoRound(object):
self.dataset = dataset
self.iters = iters
- self.multimodal = multimodal
+ self.quant_block_list = quant_block_list
if self.iters <= 0:
logger.warning("iters must be positive, reset it to 200")
self.iters = 200
@@ -235,7 +234,10 @@ class AutoRound(object):
The quantized model and layer configurations.
"""
# logger.info("cache block input")
- all_blocks = get_block_names(self.model, self.multimodal)
+ if bool(self.quant_block_list):
+ all_blocks = self.quant_block_list
+ else:
+ all_blocks = get_block_names(self.model)
if len(all_blocks) == 0:
logger.warning("could not find blocks, exit with original model")
@@ -248,6 +250,7 @@ class AutoRound(object):
self.start_time = time.time()
all_first_block_names = [block[0] for block in all_blocks]
all_inputs = self.try_cache_inter_data_gpucpu(all_first_block_names, self.nsamples, layer_names=layer_names)
+ self.model = mv_module_from_gpu(self.model, self.low_cpu_mem_usage)
for block_names in all_blocks:
inputs = all_inputs[block_names[0]]
all_inputs.pop(block_names[0])
@@ -258,17 +261,16 @@ class AutoRound(object):
self.n_samples = total_samples
if total_samples < self.train_bs:
self.train_bs = total_samples
- logger.warning(f"force the train batch size to {total_samples} ")
+ logger.warning(f"force the train batch size to {total_samples}")
- self.model = mv_module_from_gpu(self.model, self.low_cpu_mem_usage)
- torch.cuda.empty_cache()
- self.quant_blocks(
- self.model,
- inputs,
- block_names,
- nblocks=self.nblocks,
- device=self.device,
- )
+ torch.cuda.empty_cache()
+ self.quant_blocks(
+ self.model,
+ inputs,
+ block_names,
+ nblocks=self.nblocks,
+ device=self.device,
+ )
self.quant_layers(layer_names, all_inputs)
@@ -366,7 +368,7 @@ class AutoRound(object):
Returns:
None
"""
- layers_in_blocks = get_layer_names_in_block(self.model, self.supported_types, self.multimodal)
+ layers_in_blocks = get_layer_names_in_block(self.model, self.supported_types, self.quant_block_list)
keys = ["data_type", "bits", "group_size", "sym", "scale_dtype", "act_bits", "act_group_size", "act_sym",
"act_dynamic"]
for n, m in self.model.named_modules():
@@ -1132,7 +1134,7 @@ class AutoRound(object):
data_type=self.data_type,
serialization_dict=serialization_dict,
backend=backend,
- multimodal=self.multimodal,
+ quant_block_list=self.quant_block_list,
**kwargs
)
return compressed_model
@@ -1147,7 +1149,7 @@ class AutoRound(object):
return []
layer_names = []
- all_layers_in_block = get_layer_names_in_block(self.model, self.supported_types, self.multimodal)
+ all_layers_in_block = get_layer_names_in_block(self.model, self.supported_types, self.quant_block_list)
for key in self.layer_config.keys():
if key in all_layers_in_block:
@@ -1518,3 +1520,4 @@ class AutoAdamRound(AutoOPTRound):
)
+
diff --git a/auto_round/export/export_to_autogptq/export.py b/auto_round/export/export_to_autogptq/export.py
index ddc9b0c52..ffe106b86 100644
--- a/auto_round/export/export_to_autogptq/export.py
+++ b/auto_round/export/export_to_autogptq/export.py
@@ -36,8 +36,8 @@
# SOFTWARE.
import torch
-from auto_round.utils import check_to_quantized, get_block_names, get_module, logger, get_layer_names_in_block, \
- set_module
+from auto_round.utils import check_to_quantized, get_block_names,\
+ get_module, logger, get_layer_names_in_block, set_module
import copy
import json
import os
@@ -107,12 +107,16 @@ def save_quantized_as_autogptq(output_dir, inplace=True, backend="auto_gptq:exll
model = kwargs["model"]
tokenizer = kwargs["tokenizer"]
supported_types = kwargs["supported_types"]
-
+ safe_serialization = True if 'safe_serialization' not in kwargs.keys() else kwargs["safe_serialization"]
+ quant_block_list = kwargs["quant_block_list"]
logger.info("Saving quantized model to autogptq format, this may take a while...")
if tokenizer is not None:
tokenizer.save_pretrained(output_dir)
##check module quantized in block, this may have bug for mixed precision quantization
- all_blocks = get_block_names(model, multimodal=True)
+ if bool(quant_block_list):
+ all_blocks = quant_block_list
+ else:
+ all_blocks = get_block_names(model)
all_to_quantized = True
modules_in_block_to_quantize = []
for block_names in all_blocks:
@@ -201,7 +205,7 @@ def save_quantized_as_autogptq(output_dir, inplace=True, backend="auto_gptq:exll
quantization_config["modules_in_block_to_quantize"] = modules_in_block_to_quantize
if hasattr(model, "config"):
model.config.quantization_config = quantization_config
- save(model, output_dir)
+ save(model, output_dir, safe_serialization=safe_serialization)
##
@@ -233,3 +237,4 @@ def save(model: torch.nn.Module, save_dir: str, max_shard_size: str = "5GB", saf
with open(os.path.join(save_dir, config_file), "w", encoding="utf-8") as f:
json.dump(model.config.quantization_config, f, indent=2)
+
diff --git a/auto_round/export/export_to_autoround/export.py b/auto_round/export/export_to_autoround/export.py
index 13b153317..309aba660 100644
--- a/auto_round/export/export_to_autoround/export.py
+++ b/auto_round/export/export_to_autoround/export.py
@@ -165,11 +165,12 @@ def save_quantized_as_autoround(output_dir, inplace=True, backend="auto_round:ex
backend = backend.replace("auto_round", "auto_gptq")
model = kwargs["model"]
- multimodal = kwargs["multimodal"]
+ quant_block_list = kwargs["quant_block_list"]
+ safe_serialization = True if 'safe_serialization' not in kwargs.keys() else kwargs["safe_serialization"]
model = model.to(torch.float16) ##force to fp16
if not inplace:
model = copy.deepcopy(model.to("cpu"))
- layer_names_in_block = get_layer_names_in_block(model, multimodal=multimodal)
+ layer_names_in_block = get_layer_names_in_block(model, quant_block_list=quant_block_list)
layer_config = kwargs["layer_config"]
quantization_config = kwargs["serialization_dict"]
@@ -199,7 +200,6 @@ def save_quantized_as_autoround(output_dir, inplace=True, backend="auto_round:ex
quantization_config["extra_config"] = extra_config
with tctl.threadpool_limits(limits=1):
for name in layer_config.keys():
-
config = kwargs["layer_config"][name]
if config["bits"] > 8:
continue
@@ -251,7 +251,7 @@ def save_quantized_as_autoround(output_dir, inplace=True, backend="auto_round:ex
tokenizer = kwargs["tokenizer"]
if tokenizer is not None:
tokenizer.save_pretrained(output_dir)
- save(model, output_dir)
+ save(model, output_dir, safe_serialization=safe_serialization)
def save(model: nn.Module, save_dir: str, max_shard_size: str = "5GB", safe_serialization: bool = True):
@@ -281,3 +281,4 @@ def save(model: nn.Module, save_dir: str, max_shard_size: str = "5GB", safe_seri
with open(os.path.join(save_dir, config_file), "w", encoding="utf-8") as f:
json.dump(model.config.quantization_config, f, indent=2)
+
diff --git a/auto_round/export/export_to_itrex/export.py b/auto_round/export/export_to_itrex/export.py
index 50680558e..1adf65987 100644
--- a/auto_round/export/export_to_itrex/export.py
+++ b/auto_round/export/export_to_itrex/export.py
@@ -75,6 +75,7 @@ def save_quantized_as_itrex(output_dir, inplace=True, **kwargs):
enable_quanted_input = kwargs["enable_quanted_input"]
scale_dtype = kwargs["scale_dtype"]
tokenizer = kwargs["tokenizer"]
+ safe_serialization = True if 'safe_serialization' not in kwargs.keys() else kwargs["safe_serialization"]
compressed_model = pack_model(model, layer_config, inplace=inplace)
if output_dir is None:
@@ -97,7 +98,7 @@ def save_quantized_as_itrex(output_dir, inplace=True, **kwargs):
config.save_pretrained(output_dir)
quantize_config.save_pretrained(output_dir)
try:
- compressed_model.save_pretrained(output_dir, safe_serialization=True)
+ compressed_model.save_pretrained(output_dir, safe_serialization=safe_serialization)
if tokenizer is not None:
tokenizer.save_pretrained(output_dir)
logger.info("Saved config file and weights of quantized model to {}.".format(output_dir))
@@ -203,9 +204,9 @@ def pack_model(
else:
q_config = layer_config
for k, v in q_config.items():
- logger.info(f"Packing {k}")
if "float" in v["data_type"]:
continue
+ logger.info(f"Packing {k}")
dtype = v["data_type"]
num_bits = v["bits"]
group_size = v["group_size"]
@@ -259,3 +260,4 @@ def pack_model(
new_module.pack(int_weight, scale, zp, m.bias)
set_module(compressed_model, k, new_module)
return compressed_model
+
diff --git a/auto_round/special_model_handler.py b/auto_round/special_model_handler.py
index 86fcb8260..ce79cfb2a 100644
--- a/auto_round/special_model_handler.py
+++ b/auto_round/special_model_handler.py
@@ -12,12 +12,11 @@
# See the License for the specific language governing permissions and
# limitations under the License.
-from .utils import logger, torch
+import torch
share_attention_mask_tuple = ("baichuan",)
special_states_dim_tuple = ("chatglm",)
-not_share_position_ids_tuple = ("llava","phi3_v")
-
+not_share_position_ids_tuple = ("llava", "phi3_v",)
def check_share_attention_mask(model, hidden_states, attention_mask=None, **kwargs):
"""Checks if the attention mask states of the hidden states are shared in the model.
@@ -65,3 +64,4 @@ def check_not_share_position_ids(model, **kwargs):
break
return bool(is_special)
+
diff --git a/auto_round/utils.py b/auto_round/utils.py
index f7bdd1f27..2d6bd52fa 100644
--- a/auto_round/utils.py
+++ b/auto_round/utils.py
@@ -27,7 +27,6 @@ from torch.amp import autocast
from functools import lru_cache
-
@lru_cache(None)
def warning_once(self, msg: str):
self.warning(msg)
@@ -166,6 +165,7 @@ def to_device(input, device=torch.device("cpu")):
return input
+
def mv_module_from_gpu(module, low_cpu_mem_usage=False):
"""Moves module from gpu to cpu or meta if low_cpu_mem_usage is true.
@@ -232,22 +232,76 @@ def check_is_cpu(device):
return device == torch.device("cpu") or device == "cpu"
-def get_block_names(model, multimodal=False):
+def validate_modules(module_names):
+ """
+ Test a list of modules' validity.
+
+ Args:
+ modules (list of str): List of strings to be validated.
+
+ Returns:
+ bool: True if all modules have equal length or not dependent, otherwise False.
+ """
+ if not bool(module_names): # pragma: no cover
+ raise ValueError(f"Empty modules")
+ if len(module_names) < 2:
+ return True
+ split_modules = [s.split('.') for s,_ in module_names]
+ lengths = [len(parts) for parts in split_modules]
+ if len(set(lengths)) == 1: # pragma: no cover
+ return True
+ max_length = max(lengths)
+ min_length = min(lengths)
+ longest_module = next(s for s in split_modules if len(s) == max_length)
+ shortest_module = next(s for s in split_modules if len(s) == min_length)
+ shortest_module = '.'.join(shortest_module)
+ longest_module = '.'.join(longest_module)
+ # Check if the shortest name is a substring of the longest name
+ if shortest_module in longest_module: # pragma: no cover
+ raise ValueError(f"Invalid modules, at least two modules detected"\
+ " as dependent, {shortest_module} and {longest_module}")
+ return True
+
+
+def get_block_names(model):
"""Get the block names for transformers-like networks.
Args:
model: The model.
Returns:
- block_names: A list of block names.
+ block_names: A list whose elements are list of block's layer names
+ """
+ block_names = []
+ target_modules = []
+ for n, m in model.named_modules():
+ if hasattr(type(m), "__name__") and "ModuleList" in type(m).__name__:
+ target_modules.append((n, m))
+ break ## only find the first modulelist, may be not robust
+ for i,target_m in enumerate(target_modules):
+ block_names.append([])
+ for n, m in target_m[1].named_children():
+ block_names[i].append(target_m[0] + "." + n)
+ return block_names
+
+
+def get_multimodal_block_names(model, quant_vision=False):
+ """Get the multimodal model block names for transformers-like networks.
+
+ Args:
+ model: The model.
+
+ Returns:
+ block_names: A list whose elements are list of block's layer names
"""
block_names = []
target_modules = []
+ Vison_blocks_tuple = ("vision", "visual",)
for n, m in model.named_modules():
- if hasattr(type(m), "__name__") and "ModuleList" in type(m).__name__ \
- and (multimodal or ('vision' not in n and 'visual' not in n)):
- target_modules.append((n, m))
- # break ## only find the first modulelist, may be not robust
+ if hasattr(type(m), "__name__") and "ModuleList" in type(m).__name__:
+ if quant_vision or all(key not in n.lower() for key in (Vison_blocks_tuple)):
+ target_modules.append((n, m))
+ validate_modules(target_modules)
for i,target_m in enumerate(target_modules):
block_names.append([])
for n, m in target_m[1].named_children():
@@ -607,7 +661,7 @@ def check_memory_availability(device, inputs, weight, org_seqlen, org_bs):
def get_layer_names_in_block(model, supported_types=[torch.nn.Linear,
- transformers.modeling_utils.Conv1D], multimodal=False):
+ transformers.modeling_utils.Conv1D], quant_block_list=None):
"""Retrieves the names of layers within each block of the model.
Returns:
@@ -618,7 +672,10 @@ def get_layer_names_in_block(model, supported_types=[torch.nn.Linear,
if isinstance(m, tuple(supported_types)):
m.tmp_name = n
layers_in_block = []
- all_blocks = get_block_names(model, multimodal)
+ if bool(quant_block_list):
+ all_blocks = quant_block_list
+ else:
+ all_blocks = get_block_names(model)
for block_names in all_blocks:
for block_name in block_names:
block = get_module(model, block_name)
@@ -742,3 +799,4 @@ def dynamic_import_inference_linear(backend, bits, group_size, sym):
return QuantLinear
+
diff --git a/examples/language-modeling/README.md b/examples/language-modeling/README.md
index bc3fec102..f9fb00598 100644
--- a/examples/language-modeling/README.md
+++ b/examples/language-modeling/README.md
@@ -147,28 +147,32 @@ The transformers version required varies across different types of models. Here,
| Model | Transformers version |
| :----: | :----: |
-| EleutherAI/gpt-j-6b | 4.28/4.30/4.34/4.36 |
-| huggyllama/llama-7b | 4.28/4.30/4.34/4.36 |
-| meta-llama/Llama-2-7b-hf | 4.30/4.34/4.36 |
-| facebook/opt-6.7b | 4.28/4.30/4.34/4.36 |
-| tiiuae/falcon-7b | 4.28/4.30/4.34/4.36 |
+| EleutherAI/gpt-j-6b | 4.28/4.30/4.34/4.36/4.40 |
+| huggyllama/llama-7b | 4.28/4.30/4.34/4.36/4.40 |
+| meta-llama/Llama-2-7b-hf | 4.30/4.34/4.36/4.40 |
+| facebook/opt-6.7b | 4.28/4.30/4.34/4.36/4.40 |
+| tiiuae/falcon-7b | 4.28/4.30/4.34/4.36/4.40 |
| mosaicml/mpt-7b | 4.28/4.30/4.34/4.36 |
| mosaicml/mpt-7b-chat | 4.34 |
-| bigscience/bloom-7b1 | 4.28/4.30/4.34/4.36 |
-| baichuan-inc/Baichuan2-7B-Chat | 4.36 |
-| Qwen/Qwen-7B | 4.28/4.30/4.34/4.36 |
-| Qwen/Qwen1.5-7B-Chat | 4.38/4.40 |
-| THUDM/chatglm3-6b | 4.34/4.36 |
-| mistralai/Mistral-7B-v0.1 | 4.34/4.36 |
-| MBZUAI/LaMini-GPT-124M | 4.34/4.36 |
-| EleutherAI/gpt-neo-125m | 4.34 |
-| databricks/dolly-v2-3b | 4.34 |
-| stabilityai/stablelm-base-alpha-3b | 4.34 |
-| Intel/neural-chat-7b-v3 | 4.34/4.36 |
-| rinna/bilingual-gpt-neox-4b | 4.36 |
-| microsoft/phi-2 | 4.36 |
+| bigscience/bloom-7b1 | 4.28/4.30/4.34/4.36/4.40 |
+| baichuan-inc/Baichuan2-7B-Chat | 4.36/4.40 |
+| Qwen/Qwen-7B | 4.28/4.30/4.34/4.36/4.40 |
+| Qwen/Qwen1.5-7B-Chat | 4.38/4.40/4.40 |
+| THUDM/chatglm3-6b | 4.34/4.36/4.40 |
+| mistralai/Mistral-7B-v0.1 | 4.34/4.36/4.40 |
+| mistralai/Mixtral-8x7B-Instruct-v0.1 | 4.40 |
+| MBZUAI/LaMini-GPT-124M | 4.34/4.36/4.40 |
+| EleutherAI/gpt-neo-125m | 4.34/4.40 |
+| databricks/dolly-v2-3b | 4.34/4.40 |
+| stabilityai/stablelm-base-alpha-3b | 4.34/4.40 |
+| Intel/neural-chat-7b-v3 | 4.34/4.36/4.40 |
+| rinna/bilingual-gpt-neox-4b | 4.36/4.40 |
+| microsoft/phi-2 | 4.36/4.40 |
| google/gemma-7b | 4.38/4.40 |
-| Salesforce/codegen25-7b-multi | 4.33.2|
+| Salesforce/codegen25-7b-multi | 4.33.2 |
+
+
+
## Reference
@@ -188,3 +192,4 @@ If you find SignRound useful for your research, please cite our paper:
+
diff --git a/examples/language-modeling/main.py b/examples/language-modeling/main.py
index 3cd5e660d..112d8fc63 100644
--- a/examples/language-modeling/main.py
+++ b/examples/language-modeling/main.py
@@ -396,7 +396,8 @@ if __name__ == '__main__':
print(excel_name, flush=True)
eval_model(model_path=output_dir, tasks=tasks, dtype=dtype, limit=None,
eval_bs=args.eval_bs, use_accelerate=args.low_gpu_mem_usage,
- device=torch_device, excel_file=excel_name)
+ device=torch_device, excel_file=excel_name,
+ trust_remote_code=not args.disable_trust_remote_code)
if not args.disable_eval and lm_eval_version == "0.4.2":
from eval_042.evaluation import simple_evaluate
@@ -407,6 +408,7 @@ if __name__ == '__main__':
model_args = f"pretrained={eval_folder}"
else:
exit() ## does not support cpu,xpu model eval
+ model_args = model_args + f",trust_remote_code={not args.disable_trust_remote_code}"
user_model = None
if args.act_bits <= 8:
user_model = model.to(device_str)
@@ -418,3 +420,4 @@ if __name__ == '__main__':
print(make_table(res))
+
diff --git a/examples/multimodal-modeling/Llava/main.py b/examples/multimodal-modeling/Llava/main.py
index 5dc734d2b..46b6b956d 100644
--- a/examples/multimodal-modeling/Llava/main.py
+++ b/examples/multimodal-modeling/Llava/main.py
@@ -31,7 +31,7 @@ class CustomDataset(Dataset): # for llava tuning
self.tokenizer = tokenizer
self.image_processor = image_processor
self.args = args
- self.args.is_multimodal = args.do_multimodal
+ self.args.is_multimodal = args.is_multimodal
def __getitem__(self, index):
sources = self.list_data_dict[index]
@@ -158,8 +158,11 @@ if __name__ == '__main__':
parser.add_argument("--act_bits", default=32, type=int,
help="activation bits")
- parser.add_argument("--do_multimodal", action='store_true',
- help="To determine whether the preprocessing should handle multimodal component.")
+ parser.add_argument("--is_multimodal", type=bool, default=True,
+ help="To determine whether the preprocessing should handle multimodal infomations.")
+
+ parser.add_argument("--quant_vision", action='store_true',
+ help="To determine whether the quantization should handle vision component.")
# ========== Calibration Datasets =============
parser.add_argument("--mm-use-im-start-end", type=bool, default=False)
@@ -170,9 +173,6 @@ if __name__ == '__main__':
parser.add_argument("--question_file", default=None, type=str,
help="The dataset for quantization training. It can be a custom one.")
- # parser.add_argument("--dataset", default=None, type=str,
- # help="The dataset for quantization training. It can be a custom one.")
-
# ================= Evaluation Related =====================
parser.add_argument("--eval-question-file", type=str, default="tables/question.jsonl")
@@ -215,6 +215,7 @@ if __name__ == '__main__':
from auto_round import (AutoRound,
AutoAdamRound)
+ from auto_round.utils import get_multimodal_block_names
model = model.eval()
@@ -305,15 +306,17 @@ if __name__ == '__main__':
if args.quant_lm_head and args.low_gpu_mem_usage:
print(f"warning, low_gpu_mem_usage=False is strongly recommended if the whole model could be loaded to "
f"gpu")
+
+ quant_block_list = get_multimodal_block_names(model, args.quant_vision)
autoround = round(model, tokenizer, args.bits, args.group_size, sym=args.sym, batch_size=args.train_bs,
dataset=dataloader, seqlen=seqlen, nblocks=args.nblocks, iters=args.iters, lr=args.lr,
minmax_lr=args.minmax_lr, enable_quanted_input=not args.disable_quanted_input, device=device_str,
- amp=not args.disable_amp, nsamples=args.nsamples,
- low_gpu_mem_usage=args.low_gpu_mem_usage,
+ amp=not args.disable_amp, nsamples=args.nsamples, layer_config=layer_config,
+ low_gpu_mem_usage=args.low_gpu_mem_usage, scale_dtype=args.scale_dtype,
seed=args.seed, gradient_accumulate_steps=args.gradient_accumulate_steps,
- scale_dtype=args.scale_dtype, layer_config=layer_config,
- enable_minmax_tuning=not args.disable_minmax_tuning, act_bits=args.act_bits, multimodal=args.do_multimodal)
+ enable_minmax_tuning=not args.disable_minmax_tuning, act_bits=args.act_bits,
+ quant_block_list=quant_block_list)
model, _ = autoround.quantize()
model_name = args.model_name.rstrip("/")
@@ -378,3 +381,4 @@ if __name__ == '__main__':
+
diff --git a/examples/multimodal-modeling/Llava/run_autoround.sh b/examples/multimodal-modeling/Llava/run_autoround.sh
index 7f888de53..1d5e39053 100644
--- a/examples/multimodal-modeling/Llava/run_autoround.sh
+++ b/examples/multimodal-modeling/Llava/run_autoround.sh
@@ -17,3 +17,4 @@ python3 main.py \
--eval-annotation-file /path/to/textvqa/TextVQA_0.5.1_val.json \
--eval-result-file "./tmp_autoround" \
--output_dir "./tmp_autoround"
+
diff --git a/examples/multimodal-modeling/Qwen-VL/main.py b/examples/multimodal-modeling/Qwen-VL/main.py
index 4a49ab728..28544271a 100644
--- a/examples/multimodal-modeling/Qwen-VL/main.py
+++ b/examples/multimodal-modeling/Qwen-VL/main.py
@@ -290,8 +290,8 @@ if __name__ == '__main__':
parser.add_argument("--act_bits", default=32, type=int,
help="activation bits")
- parser.add_argument("--do_multimodal", action='store_true',
- help="To determine whether the preprocessing should handle multimodal component.")
+ parser.add_argument("--quant_vision", action='store_true',
+ help="To determine whether the quantization should handle vision component.")
# ========== Calibration Datasets =============
parser.add_argument("--image_folder", default="coco", type=str,
@@ -371,6 +371,7 @@ if __name__ == '__main__':
from auto_round import (AutoRound,
AutoAdamRound)
+ from auto_round.utils import get_multimodal_block_names
# model = model.eval()
seqlen = args.seqlen
@@ -439,15 +440,18 @@ if __name__ == '__main__':
if args.quant_lm_head and args.low_gpu_mem_usage:
print(f"warning, low_gpu_mem_usage=False is strongly recommended if the whole model could be loaded to "
f"gpu")
+
+ quant_block_list = get_multimodal_block_names(model, args.quant_vision)
autoround = round(model, tokenizer, args.bits, args.group_size, sym=args.sym, batch_size=args.train_bs,
dataset=dataloader, seqlen=seqlen, nblocks=args.nblocks, iters=args.iters, lr=args.lr,
- minmax_lr=args.minmax_lr, enable_quanted_input=not args.disable_quanted_input, device=device_str,
+ minmax_lr=args.minmax_lr, enable_quanted_input=not args.disable_quanted_input,
amp=not args.disable_amp, nsamples=args.nsamples,
- low_gpu_mem_usage=args.low_gpu_mem_usage,
+ low_gpu_mem_usage=args.low_gpu_mem_usage, device=device_str,
seed=args.seed, gradient_accumulate_steps=args.gradient_accumulate_steps,
scale_dtype=args.scale_dtype, layer_config=layer_config,
- enable_minmax_tuning=not args.disable_minmax_tuning, act_bits=args.act_bits, multimodal=args.do_multimodal)
+ enable_minmax_tuning=not args.disable_minmax_tuning, act_bits=args.act_bits,
+ quant_block_list=quant_block_list)
model, _ = autoround.quantize()
model_name = args.model_name.rstrip("/")
@@ -520,3 +524,4 @@ if __name__ == '__main__':
device=str(torch_device)
)
+
diff --git a/examples/multimodal-modeling/Qwen-VL/run_autoround.sh b/examples/multimodal-modeling/Qwen-VL/run_autoround.sh
index 5845fe302..d59ad510d 100644
--- a/examples/multimodal-modeling/Qwen-VL/run_autoround.sh
+++ b/examples/multimodal-modeling/Qwen-VL/run_autoround.sh
@@ -2,6 +2,8 @@
set -x
device=0
+# --quant_vision ## for vision quantization
+
CUDA_VISIBLE_DEVICES=$device \
python3 main.py \
--model_name=Qwen/Qwen-VL \
@@ -9,9 +11,9 @@ python3 main.py \
--group_size 128 \
--iters 200 \
--seqlen 512 \
---do_multimodal \
--disable_quanted_input \
--deployment_device 'auto_round' \
--image_folder /path/to/coco/images/train2017/ \
--question_file /path/to/Qwen-VL_mix665k.json \
--output_dir "./tmp_autoround"
+
diff --git a/test/test_block_names.py b/test/test_block_names.py
new file mode 100644
index 000000000..5b6218a51
--- /dev/null
+++ b/test/test_block_names.py
@@ -0,0 +1,179 @@
+import copy
+import shutil
+import sys
+import unittest
+sys.path.insert(0, ".")
+sys.path.insert(0, "..")
+import torch
+import torch.nn as nn
+from transformers import AutoModelForCausalLM, AutoTokenizer
+from auto_round import AutoRound
+
+class LLMDataLoader:
+ def __init__(self, input_size=10):
+ self.batch_size = 1
+ self.input_size = input_size
+ def __iter__(self):
+ for i in range(2):
+ yield torch.ones([1, self.input_size], dtype=torch.long)
+
+
+
+# ================= simple multimodal model =================
+class TextEncoder(nn.Module):
+ def __init__(self, input_size, hidden_size):
+ super(TextEncoder, self).__init__()
+ self.fc = nn.Linear(input_size, hidden_size)
+
+ def forward(self, x):
+ return self.fc(x)
+
+class VisionEncoder(nn.Module):
+ def __init__(self, input_size, hidden_size):
+ super(VisionEncoder, self).__init__()
+ self.fc = nn.Linear(input_size, hidden_size)
+
+ def forward(self, x):
+ return self.fc(x)
+
+class TextEncoderModuleList(nn.ModuleList):
+ pass
+
+class VisionEncoderModuleList(nn.ModuleList):
+ pass
+
+class SimpleMultimodalModel(nn.Module):
+ def __init__(self, text_input_size, image_input_size, hidden_size, num_text_encoders, num_image_encoders):
+ super(SimpleMultimodalModel, self).__init__()
+ self.text_encoders = TextEncoderModuleList([TextEncoder(text_input_size, hidden_size) for _ in range(num_text_encoders)])
+ self.vision_encoders = VisionEncoderModuleList([VisionEncoder(image_input_size, hidden_size) for _ in range(num_image_encoders)])
+ self.classifier = nn.Linear(hidden_size * 2, 1) # Binary classification task
+
+ def forward(self, text_input, image_input):
+ text_features = [encoder(text_input) for encoder in self.text_encoders]
+ image_features = [encoder(image_input) for encoder in self.vision_encoders]
+
+ # Summing the outputs of all text encoders and image encoders
+ combined_text_features = torch.stack(text_features, dim=1).sum(dim=1)
+ combined_image_features = torch.stack(image_features, dim=1).sum(dim=1)
+
+ combined_features = torch.cat((combined_text_features, combined_image_features), dim=1)
+ output = self.classifier(combined_features)
+ return output
+
+
+# ================= simple MoE model =================
+class Expert(nn.Module):
+ def __init__(self, input_size, hidden_size):
+ super(Expert, self).__init__()
+ self.fc = nn.Linear(input_size, hidden_size)
+
+ def forward(self, x):
+ return self.fc(x)
+
+class ExpertGroupModuleList(nn.ModuleList):
+ pass
+
+class ExpertModuleList(nn.ModuleList):
+ pass
+
+class NestedMoEModel(nn.Module):
+ def __init__(self, input_size, hidden_size, num_groups, experts_per_group):
+ super(NestedMoEModel, self).__init__()
+ self.expert_groups = ExpertModuleList([
+ ExpertGroupModuleList([Expert(input_size, hidden_size) for _ in range(experts_per_group)])
+ for _ in range(num_groups)
+ ])
+ self.gate = nn.Linear(input_size, num_groups * experts_per_group)
+
+ def forward(self, x):
+ gate_outputs = torch.softmax(self.gate(x), dim=1)
+ expert_outputs = []
+
+ for group in self.expert_groups:
+ group_outputs = [expert(x) for expert in group]
+ group_output_sum = torch.stack(group_outputs, dim=1).sum(dim=1)
+ expert_outputs.append(group_output_sum)
+
+ expert_outputs = torch.stack(expert_outputs, dim=1).view(x.size(0), -1)
+ output = torch.sum(gate_outputs.unsqueeze(2) * expert_outputs.unsqueeze(1), dim=1)
+ return output
+
+
+class TestQuantizationBlocks(unittest.TestCase):
+ @classmethod
+ def setUpClass(self):
+ self.model_name = "MBZUAI/LaMini-GPT-124M"
+ self.tokenizer = AutoTokenizer.from_pretrained(self.model_name, trust_remote_code=True)
+ self.llm_dataloader = LLMDataLoader()
+
+ @classmethod
+ def tearDownClass(self):
+ shutil.rmtree("./saved", ignore_errors=True)
+ shutil.rmtree("runs", ignore_errors=True)
+
+
+ def test_moe_quant(self):
+ input_size = 10
+ hidden_size = 10
+ num_groups = 2
+ experts_per_group = 2
+ self.llm_dataloader = LLMDataLoader(input_size)
+ self.model = NestedMoEModel(input_size, hidden_size, num_groups, experts_per_group)
+ from auto_round.utils import get_multimodal_block_names, get_block_names
+ llm_block_names = get_block_names(self.model)
+ all_block_names = []
+ try:
+ all_block_names = get_multimodal_block_names(self.model, quant_vision=True)
+ except:
+ pass
+ assert len(llm_block_names) != len(all_block_names)
+
+
+ def test_multimodal_quant(self):
+ num_text_encoders = 1
+ num_image_encoders = 1
+ image_input_size = 10
+ text_input_size = 10
+ hidden_size = 10
+ self.model = SimpleMultimodalModel(text_input_size, image_input_size, hidden_size, num_text_encoders, num_image_encoders)
+ from auto_round.utils import get_multimodal_block_names, get_block_names
+ llm_block_names = get_block_names(self.model)
+ block_names_wo_vision = get_multimodal_block_names(self.model, quant_vision=False)
+ block_names_with_vision = get_multimodal_block_names(self.model, quant_vision=True)
+ assert block_names_wo_vision == llm_block_names
+ assert len(block_names_wo_vision) != (block_names_with_vision)
+
+
+ def test_block_name_quant(self):
+ self.model = AutoModelForCausalLM.from_pretrained(self.model_name, torch_dtype="auto", trust_remote_code=True)
+ from auto_round.utils import get_block_names, validate_modules
+ llm_block_names = get_block_names(self.model)
+ validate_modules(llm_block_names)
+ bits, group_size, sym, train_bs = 4, 128, False, 20
+ autoround = AutoRound(
+ self.model,
+ self.tokenizer,
+ bits=bits,
+ group_size=group_size,
+ sym=sym,
+ iters=2,
+ seqlen=2,
+ train_bs=train_bs,
+ dataset=self.llm_dataloader,
+ quant_block_list=llm_block_names
+ )
+ autoround.quantize()
+ try:
+ import auto_gptq
+ except:
+ return
+ autoround.save_quantized("./saved", inplace=False, safe_serialization=False)
+ autoround.save_quantized(output_dir="./saved", inplace=False, format="auto_round")
+
+
+
+
+if __name__ == "__main__":
+ unittest.main()
+