PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度
Refactor input normalization by replaying inputs for consistent preprocessing (#1094)
Yi Liu152f1a04a0ea
diff --git a/auto_round/compressors/base.py b/auto_round/compressors/base.pyindex cc12ee09a..fb2681183 100644--- a/auto_round/compressors/base.py+++ b/auto_round/compressors/base.py@@ -20,6 +20,7 @@ import timeimport tracebackfrom collections import defaultdictfrom dataclasses import asdict, fields+from functools import partialfrom typing import Any, Callable, Optional, Unionimport accelerate@@ -96,7 +97,6 @@ from auto_round.utils import (llm_load_model,memory_monitor,mv_module_from_gpu,- normalize_input,set_amax_for_all_moe_layers,set_module,to_device,@@ -1918,6 +1918,45 @@ class BaseCompressor(object):return output+ def normalize_decoding_layer_inputs_(self, decoding_layer_inputs: list[tuple[tuple[Any, dict[str, Any]]]]):+ """+ Processes and stores decoding layer inputs for block quantization.++ This function iterates through a list of captured decoding layer calls,+ replaying them through a fake decoding layer to extract and store the+ inputs required for the decoding block in `self.inputs`. This effectively+ "normalizes" the inputs by making them accessible in a consistent format+ for subsequent quantization steps.++ Args:+ decoding_layer_inputs:+ A list of entries captured by a forward hook on the decoding layer.+ Each element is expected to be a tuple whose first item is+ `(args, kwargs)`, where `args` are the positional arguments and+ `kwargs` are the keyword arguments seen during the original+ forward pass.++ The capture hook look like:++ def input_capture_hook(module, *args, **kwargs):+ _all_module_input[module._tmp_name].append((args, kwargs))+ """+ first_block_name = self.quant_block_list[0][0]++ class _FakeDecodingLayer(torch.nn.Module):+ def forward(self, *args, **kwargs):+ return args, kwargs++ fake_layer = _FakeDecodingLayer()+ fake_layer.orig_forward = fake_layer.forward+ fake_layer.forward = partial(self._get_block_forward_func(first_block_name), fake_layer)++ self.inputs = {}+ self.last_cache_name = None+ for step_input in decoding_layer_inputs:+ args, kwargs = step_input[0]+ fake_layer(*args, **kwargs)+@torch.no_grad()def calib(self, nsamples, bs):"""Perform calibration for quantization.@@ -2346,7 +2385,6 @@ class BaseCompressor(object):def _replace_forward(self):"""Replaces the forward function."""- from functools import partialfor n, m in self.model.named_modules():if n in self.to_cached_layers and type(m) not in self.supported_types: ##block@@ -2652,7 +2690,10 @@ class BaseCompressor(object):"DiffusionCompressor","MLLMCompressor",], f"Currently, {self.__class__.__name__} does not support support quantize block with this function."- input_ids, input_others = normalize_input(inputs)+ self.normalize_decoding_layer_inputs_(inputs)+ block_inputs = self.inputs[self.quant_block_list[0][0]]+ decoding_layer_first_input_name = "hidden_states"+ input_ids, input_others = self._preprocess_block_inputs(block_inputs, decoding_layer_first_input_name)return self._quantize_block(block, input_ids, input_others, q_input, device, auto_offload)def _get_loss(@@ -2959,12 +3000,32 @@ class BaseCompressor(object):return None, output- def _split_inputs(self, inputs: dict) -> tuple[torch.Tensor, dict]:- input_ids = inputs["input_ids"]- inputs.pop("input_ids", None)+ def _split_inputs(self, inputs: dict, first_input_name: str) -> tuple[torch.Tensor, dict]:+ input_ids = inputs[first_input_name]+ inputs.pop(first_input_name, None)input_others = inputsreturn input_ids, input_others+ def _preprocess_block_inputs(self, inputs, first_input_name="input_ids"):+ input_ids, input_others = self._split_inputs(inputs, first_input_name)+ clear_memory(device_list=self.device_list)+ input_ids = to_device(input_ids, self.cache_device)+ input_others = to_device(input_others, self.cache_device)+ # As in calibration phase, we may use bf16 for calibration due to low_gpu_memory usage++ tmp_dtype = self.amp_dtype if self.amp else torch.float32+ input_ids = to_dtype(input_ids, tmp_dtype)++ for key in input_others.keys():+ if isinstance(input_others[key], torch.Tensor) and (+ input_others[key].dtype == torch.float16 or input_others[key].dtype == torch.bfloat16+ ):+ input_others[key] = input_others[key].to(tmp_dtype)+ elif isinstance(input_others[key], list):+ for i in range(len(input_others[key])):+ to_dtype(input_others[key][i], tmp_dtype)+ return input_ids, input_others+def _quantize_blocks(self,model: torch.nn.Module,@@ -2991,23 +3052,7 @@ class BaseCompressor(object):for n, m in model.named_parameters():m.requires_grad_(False)- input_ids, input_others = self._split_inputs(inputs)- clear_memory(device_list=self.device_list)- input_ids = to_device(input_ids, self.cache_device)- input_others = to_device(input_others, self.cache_device)- # As in calibration phase, we may use bf16 for calibration due to low_gpu_memory usage-- tmp_dtype = self.amp_dtype if self.amp else torch.float32- input_ids = to_dtype(input_ids, tmp_dtype)-- for key in input_others.keys():- if isinstance(input_others[key], torch.Tensor) and (- input_others[key].dtype == torch.float16 or input_others[key].dtype == torch.bfloat16- ):- input_others[key] = input_others[key].to(tmp_dtype)- elif isinstance(input_others[key], list):- for i in range(len(input_others[key])):- to_dtype(input_others[key][i], tmp_dtype)+ input_ids, input_others = self._preprocess_block_inputs(inputs)if pbar is None:pbar = tqdm(range(0, len(block_names), nblocks))diff --git a/auto_round/utils/common.py b/auto_round/utils/common.pyindex 7ef626830..bb2dfb01f 100644--- a/auto_round/utils/common.py+++ b/auto_round/utils/common.py@@ -329,20 +329,3 @@ def get_reciprocal(tensor):recip[mask] = 0.0return recip---def normalize_input(- decoding_layer_inputs: tuple[Union[list[torch.Tensor], dict, Any], Optional[dict]],-) -> Tuple[List[torch.Tensor], Dict[str, Any]]:- """Normalize the decoding layer inputs into input_ids and other inputs."""- input_ids = []- input_others = {"positional_inputs": []}- for cur_inp in decoding_layer_inputs:- input_ids.append(cur_inp[0][0][0])- for key, val in cur_inp[0][1].items():- input_others[key] = val- # Force 'use_cache' to be False- if "use_cache" in input_others and input_others["use_cache"] is True:- logger.warning_once("Forcing 'use_cache' to be False during calibration.")- input_others["use_cache"] = False- return input_ids, input_others