PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度
refine sampler (#1077)
Wenhua Cheng727e573e194e
diff --git a/auto_round/compressors/base.py b/auto_round/compressors/base.pyindex 91048eb80..7660e47c5 100644--- a/auto_round/compressors/base.py+++ b/auto_round/compressors/base.py@@ -33,6 +33,7 @@ from transformers import set_seedfrom auto_round import envsfrom auto_round.auto_scheme.gen_auto_scheme import AutoSchemefrom auto_round.compressors.utils import (+ IndexSampler,block_forward,check_need_act_calibration,check_skippable_keywords,@@ -196,7 +197,7 @@ class BaseCompressor(object):disable_opt_rtn (bool, optional): Disable RTN-mode optimization (iters=0). Defaults to False.enable_alg_ext (bool, optional): Enable algorithm extension (primarily for INT2). Defaults to False.**kwargs: Backward compatible options:- - enable_alg_ext, quant_lm_head, lr, lr_scheduler, sampler, not_use_best_mse, dynamic_max_gap,+ - enable_alg_ext, quant_lm_head, lr, lr_scheduler, not_use_best_mse, dynamic_max_gap,super_group_size, super_bits, scale_dtype ("fp16" etc.),nblocks, to_quant_block_names,enable_norm_bias_tuning, enable_quanted_input,@@ -259,7 +260,6 @@ class BaseCompressor(object):enable_minmax_tuning = kwargs.pop("enable_minmax_tuning", True)minmax_lr = kwargs.pop("minmax_lr", None)lr_scheduler = kwargs.pop("lr_scheduler", None)- sampler = kwargs.pop("sampler", "rand")not_use_best_mse = kwargs.pop("not_use_best_mse", False)dynamic_max_gap = kwargs.pop("dynamic_max_gap", -1)nblocks = kwargs.pop("nblocks", 1)@@ -350,7 +350,6 @@ class BaseCompressor(object):self.lr = lrself.minmax_lr = minmax_lr or self.lrself.enable_alg_ext = enable_alg_ext- self.sampler = samplerself.not_use_best_mse = not_use_best_mseself.dynamic_max_gap = dynamic_max_gapself.lr_scheduler = lr_scheduler@@ -2487,29 +2486,33 @@ class BaseCompressor(object):scaler = self._get_scaler() # pylint: disable=assignment-from-noneinit_loss = Nonegradient_accumulate_steps = self.batch_size # Force to low gpu- batch_size = 1 # Force to low gpu- global_batch_size = batch_size * gradient_accumulate_steps- global_batch_size = min(nsamples, global_batch_size)- if self.sampler != "rand":- whole_indices = torch.randperm(nsamples)[:global_batch_size]+total_loss = 0num_elm = 1mse_reduction = "mean"if gradient_accumulate_steps != 1:mse_reduction = "sum"mse_loss = torch.nn.MSELoss(reduction=mse_reduction).to(device)+ batch_size = 1 # Force to low gpu+ global_batch_size = self.batch_size * gradient_accumulate_steps+ global_batch_size = min(nsamples, global_batch_size)+ if gradient_accumulate_steps != 1 and not self.attention_mask:+ whole_indices = torch.arange(global_batch_size)+ if q_inputs is not None:+ num_elm = self._get_current_num_elm(q_inputs, whole_indices)+ else:+ num_elm = self._get_current_num_elm(inputs, whole_indices)++ index_sampler = IndexSampler(nsamples, global_batch_size)for i in range(self.iters):total_loss = 0- if self.sampler == "rand":- whole_indices = torch.randperm(nsamples)[:global_batch_size]- if gradient_accumulate_steps != 1:- if q_inputs is not None:- num_elm = self._get_current_num_elm(q_inputs, whole_indices)- else:- num_elm = self._get_current_num_elm(inputs, whole_indices)+ global_indices = index_sampler.next_batch()+ if self.attention_mask:+ num_elm = self._get_non_zero_cnt(self.attention_mask, global_indices)+for tmp_step in range(gradient_accumulate_steps):- indices = whole_indices[tmp_step * batch_size : (tmp_step + 1) * batch_size]+ indices = global_indices[tmp_step * batch_size : (tmp_step + 1) * batch_size]if q_inputs is not None:current_input = [q_inputs[i] for i in indices]current_input = torch.cat(current_input, dim=0).to(device)@@ -2551,7 +2554,7 @@ class BaseCompressor(object):loss = mse_loss( # pylint: disable=not-callableoutput_q.to(torch.float32), current_output.to(torch.float32))-+ num_elm = 1 if num_elm <= 0 else num_elmtotal_loss += loss.item() / num_elmself._scale_loss_and_backward(scaler, loss)@@ -2615,6 +2618,13 @@ class BaseCompressor(object):current_input_ids = [input_ids[i] for i in indices]return sum(id.numel() for id in current_input_ids)+ def _get_non_zero_cnt(self, tensor: list[torch.Tensor], indices: list[int]) -> int:+ current_tensors = [tensor[i] for i in indices]+ non_zero_cnt = 0+ for t in current_tensors:+ non_zero_cnt += torch.count_nonzero(t).item()+ return non_zero_cnt+def quantize_block(self,block: torch.nn.Module,@@ -2808,7 +2818,7 @@ class BaseCompressor(object):f"layers in the block")logger.info(dump_info)- unwrapper_block(block, {}) # TODO Quant layer should change+ unwrapper_block(block, {})mv_module_from_gpu(block)return output, output@@ -2823,11 +2833,6 @@ class BaseCompressor(object):nsamples = len(input_ids["hidden_states"])else:nsamples = len(input_ids)-- global_batch_size = self.batch_size * self.gradient_accumulate_steps- global_batch_size = min(nsamples, global_batch_size)- if self.sampler != "rand":- whole_indices = torch.randperm(nsamples)[:global_batch_size]last_best_iter = 0best_loss = torch.finfo(torch.float).maxnum_elm = 1@@ -2839,30 +2844,31 @@ class BaseCompressor(object):init_loss = Nonebest_params = {}total_loss = 0+ global_batch_size = self.batch_size * self.gradient_accumulate_steps+ global_batch_size = min(nsamples, global_batch_size)# We assume the block input and output shape is same- if self.gradient_accumulate_steps != 1:+ if self.gradient_accumulate_steps != 1 and not self.attention_mask:whole_indices = torch.arange(global_batch_size)num_elm = self._get_current_num_elm(input_ids, whole_indices)+ index_sampler = IndexSampler(nsamples, global_batch_size)+ batch_size = self.batch_sizefor i in range(self.iters):if self.enable_alg_ext and self.data_type.endswith("dq"):for n, m in block.named_modules():m.cur_iter = itotal_loss = 0- if self.sampler == "rand":- whole_indices = torch.randperm(nsamples)[:global_batch_size]+ global_indices = index_sampler.next_batch()+ if self.attention_mask:+ num_elm = self._get_non_zero_cnt(self.attention_mask, global_indices)for tmp_step in range(self.gradient_accumulate_steps):- indices = whole_indices[tmp_step * self.batch_size : (tmp_step + 1) * self.batch_size]-+ indices = global_indices[tmp_step * batch_size : (tmp_step + 1) * batch_size]current_output = self._get_current_output(output, indices)-current_output = to_device(current_output, loss_device)-output_q = self._get_current_q_output(block, input_ids, input_others, indices, device, loss_device)-loss = self._get_loss(output_q, current_output, indices, mse_loss, device)-+ num_elm = 1 if num_elm <= 0 else num_elmtotal_loss += loss.item() / num_elmif self.low_gpu_mem_usage and card_0_in_high_risk:diff --git a/auto_round/compressors/utils.py b/auto_round/compressors/utils.pyindex 2514c9f34..9c6c02877 100644--- a/auto_round/compressors/utils.py+++ b/auto_round/compressors/utils.py@@ -13,6 +13,7 @@# limitations under the License.import copyimport os+import randomimport reimport sysfrom dataclasses import asdict, fields@@ -1310,3 +1311,56 @@ def immediate_saving(rounder: object, m: torch.nn.Module, name: str = None, lastclear_memory()except Exception as _cleanup_err:logger.warning(f"shard cleanup warning: {_cleanup_err}")+++class IndexSampler:+ """A cyclic sampler that returns shuffled index batches.++ This sampler maintains internal state so that each call to `next_batch()`+ continues from where it left off. When the remaining number of samples is+ less than `batch_size`, the sampler reshuffles all indices and starts from+ the beginning, discarding the last incomplete batch.++ Attributes:+ nsamples (int): Total number of samples.+ batch_size (int): Number of indices to return in each batch.+ index (int): Current position in the index list.+ indices (List[int]): Shuffled list of indices.+ """++ def __init__(self, nsamples: int, batch_size: int) -> None:+ """Initializes the sampler.++ Args:+ nsamples (int): Total number of samples (must be >= batch_size).+ batch_size (int): Number of indices per batch.++ Raises:+ ValueError: If batch_size is not in the range (0, nsamples].+ """+ if batch_size <= 0 or batch_size > nsamples:+ raise ValueError("batch_size must be > 0 and <= nsamples")++ self.nsamples: int = nsamples+ self.batch_size: int = batch_size+ self.index: int = 0++ self.indices: list[int] = list(range(nsamples))+ random.shuffle(self.indices)++ def next_batch(self) -> list[int]:+ """Returns the next batch of shuffled indices.++ If the remaining indices are fewer than `batch_size`, the sampler+ reshuffles the entire list and starts from the beginning.++ Returns:+ list[int]: A list of size `batch_size` containing sample indices.+ """+ if self.index + self.batch_size > self.nsamples:+ random.shuffle(self.indices)+ self.index = 0++ batch = self.indices[self.index : self.index + self.batch_size]+ self.index += self.batch_size+ return batchdiff --git a/auto_round/utils/common.py b/auto_round/utils/common.pyindex 3241f0cb1..7ef626830 100644--- a/auto_round/utils/common.py+++ b/auto_round/utils/common.py@@ -17,7 +17,7 @@ import importlibimport osimport reimport sys-from typing import Any, Callable, Dict, List, Tuple, Union+from typing import Any, Callable, Dict, List, Optional, Tuple, Unionimport torchimport transformers@@ -331,11 +331,12 @@ def get_reciprocal(tensor):return recip-def normalize_input(decoding_layer_inputs: list[tuple[Any]]) -> Tuple[List[torch.Tensor], Dict[str, Any]]:+def normalize_input(+ decoding_layer_inputs: tuple[Union[list[torch.Tensor], dict, Any], Optional[dict]],+) -> Tuple[List[torch.Tensor], Dict[str, Any]]:"""Normalize the decoding layer inputs into input_ids and other inputs."""input_ids = []- input_others = {}- input_others["positional_inputs"] = []+ input_others = {"positional_inputs": []}for cur_inp in decoding_layer_inputs:input_ids.append(cur_inp[0][0][0])for key, val in cur_inp[0][1].items():