PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度
fix bs!=1 for gemma and MiniMax-Text-01 (#515)
Wenhua Cheng23cd60dd541e
diff --git a/README.md b/README.mdindex 33048db91..53a5b7171 100644--- a/README.md+++ b/README.md@@ -44,7 +44,7 @@ and [fbaldassarri](https://huggingface.co/fbaldassarri).```bash# GPU-pip install auto-round[gpu]+pip install auto-round# CPUpip install auto-round[cpu]@@ -58,7 +58,7 @@ pip install auto-round-lib```bash# GPU- pip install .[gpu]+ pip install .# CPUpip install .[cpu]@@ -102,8 +102,8 @@ auto-round-best \``````bash-auto-round-light \## light accuracy, 2-3X speedup, slight accuracy drop at W4 and larger accuracy drop at W2+auto-round-light \--model facebook/opt-125m \--bits 4 \--group_size 128 \diff --git a/auto_round/autoround.py b/auto_round/autoround.pyindex 4260e6720..af175fa18 100644--- a/auto_round/autoround.py+++ b/auto_round/autoround.py@@ -25,12 +25,6 @@ from torch import autocastfrom tqdm import tqdmimport acceleratefrom .wrapper import WrapperMultiblock, wrapper_block, unwrapper_block, WrapperLinear, unwrapper_layer-from .special_model_handler import (- shareable_keywords,- init_cache_for_special_model,- reset_params,- check_skippable_keywords-)from .utils import (CpuInfo,block_forward,@@ -44,7 +38,6 @@ from .utils import (htcore,is_optimum_habana_available,logger,- sampling_inputs,to_device,to_dtype,get_layer_names_in_block,@@ -54,9 +47,11 @@ from .utils import (find_matching_blocks, is_debug_mode,TORCH_VERSION_AT_LEAST_2_6,supported_layer_types,- get_layer_features,+ get_layer_features,set_module,- llm_load_model+ llm_load_model,+ reset_params,+ init_cache, check_skippable_keywords, get_shared_keys,)from .low_cpu_mem.utils import get_layers_before_block@@ -311,6 +306,7 @@ class AutoRound(object):]self.has_qlayer_outside_block = self.set_layerwise_config(self.layer_config) ##better place in the end+ self.shared_cache_keys = get_shared_keys(self.model)def set_device_map_in_blocks(self, device_map):"""Sets the device map for specific blocks in the model.@@ -773,12 +769,13 @@ class AutoRound(object):for i in range(0, nsamples, bs):end_index = min(nsamples, i + bs)indices = torch.arange(i, end_index).to(torch.long)- tmp_input_ids, tmp_input_others = sampling_inputs(+ tmp_input_ids, tmp_input_others = AutoRound.sampling_inputs(input_ids,input_others,indices,self.seqlen,- self.batch_dim+ self.batch_dim,+ share_cache_keys=self.shared_cache_keys)tmp_output = block_forward(block, tmp_input_ids, tmp_input_others, self.amp, self.amp_dtype, device).to(cache_device@@ -1010,7 +1007,7 @@ class AutoRound(object):new_data = dataif batch_size <= 1:return new_data- if data_name in shareable_keywords:+ if data_name in self.shared_cache_keys:return Noneif "alibi" in data_name:if isinstance(data, torch.Tensor):@@ -1032,7 +1029,7 @@ class AutoRound(object):"""if name not in self.inputs:self.inputs[name] = {}- init_cache_for_special_model(self.model, positional_inputs, self.inputs[name])+ init_cache(positional_inputs, self.inputs[name])if self.batch_dim is None:self.batch_dim = 0@@ -1055,7 +1052,7 @@ class AutoRound(object):or isinstance(kwargs[key], tuple):if key not in self.inputs[name].keys(): # initializationdata = to_device(kwargs[key], device=torch.device("cpu"))- if data is None or (self.batch_size > 1 and key in shareable_keywords):+ if data is None or (self.batch_size > 1 and key in self.shared_cache_keys):self.inputs[name][key] = datacontinueif self.batch_size <= 1:@@ -1174,7 +1171,7 @@ class AutoRound(object):if self.lr_scheduler is None:lr_schedule = torch.optim.lr_scheduler.LinearLR(- optimizer, start_factor=1.0, end_factor=0.0, total_iters=self.iters, verbose=False+ optimizer, start_factor=1.0, end_factor=0.0, total_iters=self.iters)else:lr_schedule = copy.deepcopy(self.lr_scheduler)@@ -1357,7 +1354,7 @@ class AutoRound(object):if self.lr_scheduler is None:lr_schedule = torch.optim.lr_scheduler.LinearLR(- optimizer, start_factor=1.0, end_factor=0.0, total_iters=self.iters, verbose=False+ optimizer, start_factor=1.0, end_factor=0.0, total_iters=self.iters)else:lr_schedule = copy.deepcopy(self.lr_scheduler)@@ -1389,12 +1386,13 @@ class AutoRound(object):num_elm = sum(id.numel() for id in current_input_ids)for tmp_step in range(self.gradient_accumulate_steps):indices = whole_indices[tmp_step * self.batch_size: (tmp_step + 1) * self.batch_size]- current_input_ids, current_input_others = sampling_inputs(+ current_input_ids, current_input_others = AutoRound.sampling_inputs(input_ids,input_others,indices,seqlen=self.seqlen,batch_dim=self.batch_dim,+ share_cache_keys=self.shared_cache_keys)current_output = [output[x] for x in indices]@@ -1786,6 +1784,47 @@ class AutoRound(object):optimizer.zero_grad()lr_schedule.step()+ @classmethod+ @torch.no_grad()+ def sampling_inputs(cls, input_ids, input_others, indices, seqlen,+ batch_dim=0,share_cache_keys=()):+ """Samples inputs based on the given indices and sequence length.++ Args:+ input_ids: The list of input tensor containing input_ids.+ input_others: A dictionary containing other input data.+ indices: The indices to sample from the input.+ seqlen: The sequence length.++ Returns:+ current_input_ids: The sampled input IDs.+ current_input_others: The sampled other input data.+ """+ current_input_ids = [input_ids[i] for i in indices]++ current_input_ids = torch.cat(current_input_ids, dim=batch_dim)++ current_input_others = {"positional_inputs": input_others["positional_inputs"]}+ for key in input_others.keys():+ if "positional_inputs" in key:+ continue+ if (key not in share_cache_keys or len(indices) == 1) \+ and not isinstance(input_others[key], (str, bool, type(None))):+ current_input_others[key] = None+ if input_others[key] is not None:+ current_input_others[key] = [input_others[key][i] for i in indices]+ if len(indices) == 1:+ current_input_others[key] = current_input_others[key][0]+ else:+ try:+ current_input_others[key] = torch.cat(current_input_others[key], dim=0)+ except TypeError as err:+ logger.warning_once("Please check the model cache inputs or try setting batch_size to 1.")+ else:+ current_input_others[key] = input_others[key]++ return current_input_ids, current_input_others+class AutoRoundOPT(AutoRound):"""Class for automatic rounding-based quantization with optimizers like adamw of a PyTorch model.diff --git a/auto_round/special_model_handler.py b/auto_round/special_model_handler.pyindex de8eb4ea7..f602f9b3f 100644--- a/auto_round/special_model_handler.py+++ b/auto_round/special_model_handler.py@@ -12,11 +12,8 @@# See the License for the specific language governing permissions and# limitations under the License.-import torch-from collections import UserDict-shareable_keywords = ("position_ids", "cache_position", "position_embeddings")-mllms_with_limited_bs = ("llava", "qwen2_vl", "phi3_v", "mllama") # Limitations on batch_size-skippable_cache_keys = ("past_key_value",)++mllms_with_limited_bs = ("llava", "qwen2_vl", "phi3_v", "mllama") # Limitations on batch_sizeSUPPORT_ONLY_TEXT_MODELS = ["phi3_v",@@ -28,9 +25,13 @@ SUPPORT_ONLY_TEXT_MODELS = ["idefics3"]+SPECIAL_SHARED_CACHE_KEYS = {+ "Gemma3ForConditionalGeneration": ("position_embeddings_global", "position_embeddings_local")}+SPECIAL_SHARED_CACHE_KEYS["MiniMaxText01ForCausalLM"] = ("slope_rate",)++def _handle_special_model(model):if model.config.model_type == "deepseek_vl_v2":- from auto_round.special_model_handler import _deepseek_vl2_forwardfrom functools import partialmodel.forward = partial(_deepseek_vl2_forward, model)return model@@ -45,31 +46,33 @@ def _get_deepseek_vl2_multimodal_block(model, quant_vision=False):block_names.append([f"language.model.layers.{i}" for i in range(len(model.language.model.layers))])return block_names+SPECIAL_MULTIMODAL_BLOCK = {"deepseek_vl_v2": _get_deepseek_vl2_multimodal_block}+def _deepseek_vl2_forward(- model,- input_ids = None,-- position_ids = None,- attention_mask = None,- past_key_values = None,- inputs_embeds = None,-- images = None,- images_seq_mask = None,- images_spatial_crop = None,-- labels = None,- use_cache = None,- output_attentions = None,- output_hidden_states = None,- return_dict = None,- cache_position = None,- **kwargs- ):+ model,+ input_ids=None,++ position_ids=None,+ attention_mask=None,+ past_key_values=None,+ inputs_embeds=None,++ images=None,+ images_seq_mask=None,+ images_spatial_crop=None,++ labels=None,+ use_cache=None,+ output_attentions=None,+ output_hidden_states=None,+ return_dict=None,+ cache_position=None,+ **kwargs+):inputs_embeds = model.prepare_inputs_embeds(input_ids=input_ids,images=images,@@ -90,79 +93,6 @@ def _deepseek_vl2_forward(cache_position=cache_position)-def to_device(input, device=torch.device("cpu")):- """Moves input data to the specified device.-- Args:- input: The input data to be moved.- device: The target device.-- Returns:- The input data on the specified device.- """- if input is None:- return None- if isinstance(input, torch.Tensor):- return input.to(device)- if isinstance(input, dict) or isinstance(input, UserDict):- for inp in input.keys():- input[inp] = to_device(input[inp], device)-- elif isinstance(input, list) or isinstance(input, tuple):- if len(input) == 0:- return input- input_res = []- for inp in input:- input_res.append(to_device(inp, device))- if isinstance(input, tuple):- input_res = tuple(input_res)- input = input_res-- return input---def init_cache_for_special_model(model, positional_inputs, inputs):- """- Initializes special model inputs by adding positional inputs if missing.-- Args:- model: The model instance being initialized.- positional_inputs (list): List of positional inputs to add to inputs.- inputs (dict): Dictionary of model inputs.-- Modifies:- inputs (dict): Adds "positional_inputs" key if not present.- """- if "positional_inputs" not in inputs: # for chatglm Series- inputs["positional_inputs"] = []- for idx, item in enumerate(positional_inputs):- inputs["positional_inputs"] = to_device(positional_inputs)---def reset_params(inputs):- """- Resets specific input parameters to avoid saving the key-value cache during fine-tuning.-- Args:- inputs (dict): Dictionary of model inputs.-- Modifies:- inputs (dict): Sets "use_cache" to False if the key is present.- """- if "use_cache" in inputs.keys(): # Not storing kv cache- inputs['use_cache'] = False---def check_skippable_keywords(key):- """- Prints a reminder if a key is not stored during quantization fine-tuning.- """- for cache_key in skippable_cache_keys:- if cache_key not in key:- return True- return False--def check_mllm_model_batch(model, batch_size, gradient_accumulate_steps=1):"""Checks model configuration to determine if it's necessary to limit bs to avoid potential input shape mismatches.@@ -171,6 +101,6 @@ def check_mllm_model_batch(model, batch_size, gradient_accumulate_steps=1):if hasattr(model, "config") and key in model.config.model_type and batch_size != 1:accumulate_steps = batch_size * gradient_accumulate_stepsprint("To avoid the tensor concat mismatch problem, modified parameters to " \- f"batch_size=1. As an alternative, set the gradient_accumulate_steps={accumulate_steps}")+ f"batch_size=1. As an alternative, set the gradient_accumulate_steps={accumulate_steps}")return 1, accumulate_stepsreturn batch_size, gradient_accumulate_stepsdiff --git a/auto_round/utils.py b/auto_round/utils.pyindex ea9779893..00db00aae 100644--- a/auto_round/utils.py+++ b/auto_round/utils.py@@ -27,10 +27,12 @@ from torch.amp import autocastfrom functools import lru_cachefrom packaging import versionimport gc-from .special_model_handler import shareable_keywords, SPECIAL_MULTIMODAL_BLOCK+from .special_model_handler import SPECIAL_MULTIMODAL_BLOCK, SPECIAL_SHARED_CACHE_KEYSimport transformersfrom auto_round.export.export_to_gguf.config import GGUF_CONFIG+shared_cache_keys = ("position_ids", "cache_position", "position_embeddings")+supported_formats = ("auto_round", "auto_gptq", "auto_awq", "auto_round:auto_gptq", "auto_round:gptqmodel", "auto_round:auto_awq","itrex", "itrex_xpu", "fake"@@ -389,6 +391,7 @@ def get_block_names(model, quant_vision=False):Returns:block_names: A list whose elements are list of block's layer names"""+def _get_llm_block_names(model):block_names = []target_modules = []@@ -438,47 +441,6 @@ def collect_best_params(block):return params[email protected]_grad()-def sampling_inputs(input_ids, input_others, indices, seqlen,- batch_dim=0):- """Samples inputs based on the given indices and sequence length.-- Args:- input_ids: The list of input tensor containing input_ids.- input_others: A dictionary containing other input data.- indices: The indices to sample from the input.- seqlen: The sequence length.-- Returns:- current_input_ids: The sampled input IDs.- current_input_others: The sampled other input data.- """- current_input_ids = [input_ids[i] for i in indices]-- current_input_ids = torch.cat(current_input_ids, dim=batch_dim)-- current_input_others = {"positional_inputs": input_others["positional_inputs"]}- for key in input_others.keys():- if "positional_inputs" in key:- continue- if (key not in shareable_keywords or len(indices) == 1) \- and not isinstance(input_others[key], (str, bool, type(None))):- current_input_others[key] = None- if input_others[key] is not None:- current_input_others[key] = [input_others[key][i] for i in indices]- if len(indices) == 1:- current_input_others[key] = current_input_others[key][0]- else:- try:- current_input_others[key] = torch.cat(current_input_others[key], dim=0)- except TypeError as err:- logger.warning_once("Please check the model cache inputs or try setting batch_size to 1.")- else:- current_input_others[key] = input_others[key]-- return current_input_ids, current_input_others--def block_forward(block, input_ids, input_others, amp=False, amp_dtype=torch.float16, device=torch.device("cpu")):"""Performs a forward pass through a block with the given inputs.@@ -1204,7 +1166,6 @@ def is_debug_mode():return sys.gettrace() is not None or sys.flags.debug == 1-def get_layer_features(layer):"""Extracts input and output feature dimensions for supported layers."""if isinstance(layer, torch.nn.Linear):@@ -1213,12 +1174,13 @@ def get_layer_features(layer):return layer.weight.shape[0], layer.weight.shape[1]return None, None # Unsupported layer type+def _gguf_args_check(args):from auto_round.utils import loggerfrom auto_round.export.export_to_gguf.config import GGUF_CONFIGformats = args.format.lower().replace(' ', '').split(",")- formats = sorted(formats, key=lambda x:len(x))+ formats = sorted(formats, key=lambda x: len(x))pattern = re.compile("q\d_k")pre_dq_format = ""for format in GGUF_CONFIG:@@ -1241,7 +1203,7 @@ def _gguf_args_check(args):logger.error(f"Model {model_architecture} is not supported to export GGUF format")sys.exit(1)- if re.search(pattern, format) and ("hidden_size" in hparams and hparams["hidden_size"] % 256 !=0):+ if re.search(pattern, format) and ("hidden_size" in hparams and hparams["hidden_size"] % 256 != 0):model_name = args.model.split('/')model_name = model_name[-1] if model_name[-1] else model_name[-2]hidden_size = hparams["hidden_size"]@@ -1267,7 +1229,7 @@ def _gguf_args_check(args):def _to_model_dtype(model, model_dtype):- if model_dtype != None:+ if model_dtype is not None:try:if model_dtype == "float16" or model_dtype == "fp16":model = model.to(torch.float16)@@ -1280,6 +1242,7 @@ def _to_model_dtype(model, model_dtype):exit()return model+def llm_load_model(pretrained_model_name_or_path,torch_dtype="auto",@@ -1288,7 +1251,7 @@ def llm_load_model(model_dtype=None,device="cpu",low_cpu_mem_mode=0,- low_cpu_mem_tmp_dir = None,+ low_cpu_mem_tmp_dir=None,**kwargs):from transformers import AutoTokenizer, AutoModel, AutoModelForCausalLM@@ -1322,7 +1285,6 @@ def llm_load_model(torch_dtype=torch_dtype,trust_remote_code=trust_remote_code)else:- from auto_round.utils import _use_hpu_compile_modeif _use_hpu_compile_mode():model = model_cls.from_pretrained(pretrained_model_name_or_path, low_cpu_mem_usage=True, torch_dtype=torch_dtype,@@ -1403,9 +1365,10 @@ def mllm_load_model(return model, processor, tokenizer, image_processor+def is_pure_text_model(model):"""verify on: phi-3.5, Mistral-Small-3.1, gemma-3, qwen2-vl, """- if hasattr(model,"config") and hasattr(model.config, "vision_config"):+ if hasattr(model, "config") and hasattr(model.config, "vision_config"):return Falseif hasattr(model.__class__, "main_input_name") and model.__class__.main_input_name != "input_ids":return False@@ -1419,3 +1382,60 @@ def is_pure_text_model(model):if "img" in str(module.__class__).lower():return Falsereturn True+++def reset_params(inputs):+ """+ Resets specific input parameters to avoid saving the key-value cache during fine-tuning.++ Args:+ inputs (dict): Dictionary of model inputs.++ Modifies:+ inputs (dict): Sets "use_cache" to False if the key is present.+ """+ if "use_cache" in inputs.keys(): # Not storing kv cache+ inputs['use_cache'] = False+++def check_skippable_keywords(key):+ """+ Prints a reminder if a key is not stored during quantization fine-tuning.+ """+ skippable_cache_keys = ("past_key_value",)+ for cache_key in skippable_cache_keys:+ if cache_key not in key:+ return True+ return False+++def init_cache(positional_inputs, inputs):+ """+ Initializes special model inputs by adding positional inputs if missing.++ Args:+ positional_inputs (list): List of positional inputs to add to inputs.+ inputs (dict): Dictionary of model inputs.++ Modifies:+ inputs (dict): Adds "positional_inputs" key if not present.+ """+ if "positional_inputs" not in inputs: # for chatglm Series+ inputs["positional_inputs"] = []+ for idx, item in enumerate(positional_inputs):+ inputs["positional_inputs"] = to_device(positional_inputs)+++def get_shared_keys(model):+ """+ Retrieves shared keys from the model's state dictionary.++ Args:+ model (torch.nn.Module): The model to retrieve shared keys from.++ Returns:+ tuple: tuple of shared keys.+ """+ shared_keys = shared_cache_keys+ shared_keys += SPECIAL_SHARED_CACHE_KEYS.get(model.__class__.__name__, ())+ return shared_keysdiff --git a/auto_round_extension/cuda/triton_utils/dequant.py b/auto_round_extension/cuda/triton_utils/dequant.pyindex d971059d3..524612489 100644--- a/auto_round_extension/cuda/triton_utils/dequant.py+++ b/auto_round_extension/cuda/triton_utils/dequant.py@@ -162,7 +162,6 @@ def quant_matmul_248(class QuantLinearFunction(torch.autograd.Function):@staticmethod- @custom_fwddef forward(ctx, input, qweight, scales, qzeros, g_idx, bits, maxq):output = quant_matmul_248(input, qweight, scales, qzeros, g_idx, bits, maxq)ctx.save_for_backward(qweight, scales, qzeros, g_idx)@@ -170,7 +169,6 @@ class QuantLinearFunction(torch.autograd.Function):return output@staticmethod- @custom_bwddef backward(ctx, grad_output):qweight, scales, qzeros, g_idx = ctx.saved_tensorsbits, maxq = ctx.bits, ctx.maxqdiff --git a/auto_round_extension/cuda/triton_utils_zp/dequant.py b/auto_round_extension/cuda/triton_utils_zp/dequant.pyindex 4ef9936ce..416a406eb 100644--- a/auto_round_extension/cuda/triton_utils_zp/dequant.py+++ b/auto_round_extension/cuda/triton_utils_zp/dequant.py@@ -162,7 +162,6 @@ def quant_matmul_248(class QuantLinearFunction(torch.autograd.Function):@staticmethod- @custom_fwddef forward(ctx, input, qweight, scales, qzeros, g_idx, bits, maxq):output = quant_matmul_248(input, qweight, scales, qzeros, g_idx, bits, maxq)ctx.save_for_backward(qweight, scales, qzeros, g_idx)@@ -170,7 +169,6 @@ class QuantLinearFunction(torch.autograd.Function):return output@staticmethod- @custom_bwddef backward(ctx, grad_output):qweight, scales, qzeros, g_idx = ctx.saved_tensorsbits, maxq = ctx.bits, ctx.maxqdiff --git a/requirements-gpu.txt b/requirements-gpu.txtdeleted file mode 100644index 4a5350e73..000000000--- a/requirements-gpu.txt+++ /dev/null@@ -1 +0,0 @@-auto-gptq>=0.7.1diff --git a/setup.py b/setup.pyindex 3b8ec0222..a725e32e6 100644--- a/setup.py+++ b/setup.py@@ -82,7 +82,6 @@ PKG_INSTALL_CFG = {),"install_requires": fetch_requirements("requirements.txt"),"extras_require": {- "gpu": fetch_requirements("requirements-gpu.txt"),"cpu": fetch_requirements("requirements-cpu.txt"),},}