PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度
refine code (#749)
Wenhua Chenge9b2b83bb8d7
diff --git a/auto_round/__init__.py b/auto_round/__init__.pyindex 7d5215f68..93cce6299 100644--- a/auto_round/__init__.py+++ b/auto_round/__init__.py@@ -11,7 +11,7 @@# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.# See the License for the specific language governing permissions and# limitations under the License.-from auto_round.autoround import AutoRound, AutoRoundAdam, AutoRoundOPT+from auto_round.autoround import AutoRound, AutoRoundAdamfrom auto_round.mllm import AutoRoundMLLMfrom auto_round.utils import LazyImportdiff --git a/auto_round/autoround.py b/auto_round/autoround.pyindex 8faf3f9b8..20d01241d 100644--- a/auto_round/autoround.py+++ b/auto_round/autoround.py@@ -83,69 +83,21 @@ from auto_round.wrapper import WrapperLinear, WrapperMultiblock, unwrapper_blockclass AutoRound(object):- """For more information, please refer to Cheng, Wenhua, et al. "Optimize weight rounding via signed gradient descent- for the quantization of llms." arXiv preprint arXiv:2309.05516 (2023).-- Args:- model: The PyTorch model to be quantized.- tokenizer: An optional tokenizer for processing input data. If none is provided, a dataloader must be supplied.- bits (int): Number of bits for quantization (default is 4).- group_size (int): Size of the quantization group (default is 128).- sym (bool): Whether symmetric quantization is to be used (default is True).- layer_config (dict): Configuration for weight quantization (default is None).- layer_config={- 'layer1':##layer_name- {- 'data_type': 'int',- 'bits': 4,- 'group_size': 128,- 'sym': True- 'act_data_type': None,- 'act_bits': 16,- 'act_group_size': None,- 'act_sym': None,-- }- ...- }- batch_size (int): Batch size for training (default is 8).- amp (bool): Whether to use automatic mixed precision (default is True).- device: The device to be used for tuning (default is "auto").- lr_scheduler: The learning rate scheduler to be used.- dataset (str): The default dataset name (default is "NeelNanda/pile-10k").- enable_quanted_input (bool): Whether to use the output of the previous quantized block as- the input for the current block (default is True).- enable_minmax_tuning (bool): Whether to enable weight min-max tuning (default is True).- lr (float): The learning rate (default is None, will be set to 1.0/iters).- minmax_lr (float): The learning rate for min-max tuning (default is None, it will be set to lr automatically).- low_gpu_mem_usage (bool): Whether to use low GPU memory (default is True).- low_cpu_mem_usage (bool): Whether to use low CPU memory (default is False).- iters (int): Number of iterations (default is 200).- seqlen (int): Data length of the sequence for tuning (default is 2048).- nsamples (int): Number of samples (default is 128).- sampler (str): The sampling method (default is "rand").- seed (int): The random seed (default is 42).- nblocks (int): Number of blocks (default is 1).- gradient_accumulate_steps (int): Number of gradient accumulation steps (default is 1).- not_use_best_mse (bool): Whether to use mean squared error (default is False).- dynamic_max_gap (int): The dynamic maximum gap (default is -1).- data_type (str): The data type to be used (default is "int").- scale_dtype (str): The data type of quantization scale to be used (default is "float16"), different kernels- have different choices.- act_bits (int): Number of bits for activation quantization. Default is 16.- act_group_size (int): Group size for activation quantization. Default is None.- act_sym (bool): Whether to use symmetric activation quantization. Default is None.- act_data_type (str): Specifies the data type for activations.- Defaults to None, in which case it inherits the weight data type.- act_dynamic (bool): Whether to use dynamic activation quantization. Default is True.- to_quant_block_names (str|list): A string or list whose elements are list of- block's layer names to be quantized.- enable_norm_bias_tuning (bool): Whether to enable fast norm/layer_bias tuning- enable_torch_compile (bool): Whether to enable torch compile to optimize quant_block/layer (default it False).- device_map (str|dict): device map for each block- disable_opt_rtn (bool): Whether to disable optimization of the RTN mode(iters=0) (default is False).- Returns:- The quantized model.+ """Automatic weight rounding (Signed Gradient Descent) for LLM quantization++ Reference:+ Cheng, Wenhua, et al., "Optimize weight rounding via signed gradient descent for+ the quantization of LLMs." arXiv:2309.05516 (2023).++ Attributes:+ model (torch.nn.Module): The loaded PyTorch model in eval mode.+ tokenizer: Tokenizer used to prepare input text for calibration/tuning.+ bits (int): Weight quantization bits.+ group_size (int): Per-group size for weight quantization.+ sym (bool): Whether to use symmetric weight quantization.+ layer_config (dict): Per-layer quantization configuration.+ nsamples (int): Number of calibration samples.+ enable_torch_compile (bool): Whether to enable torch.compile for quant blocks/layers."""def __init__(@@ -181,8 +133,66 @@ class AutoRound(object):enable_alg_ext: bool = False,**kwargs,):- ## to ensure backward compatibility, move infrequently used arguments to kwargs arguments.- ## major version releases may be pack them with extra configuration options+ """Initialize AutoRound with quantization and tuning configuration.++ Args:+ model (torch.nn.Module | str): Model object or model name to load.+ tokenizer: Tokenizer for text processing. Required if `model` is not a string and `iters > 0`.+ bits (int, optional): Weight quantization bits. Defaults to 4.+ group_size (int, optional): Weight quantization group size. Defaults to 128.+ sym (bool, optional): Symmetric weight quantization. Defaults to True.+ layer_config (dict, optional): Layer-wise quantization config. Defaults to None.+ batch_size (int, optional): Calibration batch size. Defaults to 8.+ amp (bool, optional): Use AMP for tuning. Defaults to True.+ device (str | torch.device | int, optional): Compute device. Defaults to 0.+ dataset (str | list | tuple | DataLoader, optional): Calibration data. Defaults to "NeelNanda/pile-10k".+ enable_minmax_tuning (bool, optional): Enable weight min-max tuning. Defaults to True.+ lr (float, optional): Learning rate; if None, set to 1.0 / iters except when iters==0.+ minmax_lr (float, optional): Learning rate for min-max tuning; defaults to `lr`.+ low_gpu_mem_usage (bool, optional): Lower GPU memory mode. Defaults to False.+ iters (int, optional): Optimization iterations. Defaults to 200.+ seqlen (int, optional): Calibration sequence length. Defaults to 2048.+ nsamples (int, optional): Number of calibration samples. Defaults to 128.+ seed (int, optional): Random seed. Defaults to 42.+ gradient_accumulate_steps (int, optional): Gradient accumulation steps. Defaults to 1.+ data_type (str, optional): Weight data type string, e.g., "int". Defaults to "int".+ act_bits (int, optional): Activation quantization bits. Defaults to 16.+ act_group_size (int, optional): Activation group size. Defaults to None.+ act_sym (bool, optional): Symmetric activation quantization. Defaults to None.+ act_data_type (str, optional): Activation data type; inherits weight dtype if None and act_bits < 16.+ act_dynamic (bool, optional): Dynamic activation quantization. Defaults to True.+ enable_torch_compile (bool, optional): Enable torch.compile for quant blocks/layers. Defaults to False.+ device_map (str | dict, optional): Device placement map. Defaults to None.+ disable_opt_rtn (bool, optional): Disable RTN-mode optimization (iters=0). Defaults to False.+ enable_alg_ext (bool, optional): Enable algorithm extension (primarily for INT2). Defaults to False.+ **kwargs: Backward compatible options:+ - lr_scheduler, sampler, not_use_best_mse, dynamic_max_gap,+ super_group_size, super_bits, scale_dtype ("fp16" etc.),+ nblocks, low_cpu_mem_usage, to_quant_block_names,+ enable_norm_bias_tuning, enable_quanted_input,+ disable_deterministic_algorithms, vlm, static_kv_dtype+ Raises:+ ValueError: If invalid device is provided or tokenizer is missing for non-str model with iters > 0.+ RuntimeError: If model parameters are on meta device.+ Example:+ Layer-wise configuration structure:++ >>> layer_config = {+ ... "layer1": {+ ... "data_type": "int",+ ... "bits": 4,+ ... "group_size": 128,+ ... "sym": True,+ ... "act_data_type": None,+ ... "act_bits": 16,+ ... "act_group_size": None,+ ... "act_sym": None,+ ... },+ ... # ...+ ... }+ """+ # Extra/legacy kwargs for backward compatibility+ # Major version releases may pack them with extra configuration optionslr_scheduler = kwargs.pop("lr_scheduler", None)sampler = kwargs.pop("sampler", "rand")not_use_best_mse = kwargs.pop("not_use_best_mse", False)@@ -196,91 +206,41 @@ class AutoRound(object):enable_norm_bias_tuning: bool = kwargs.pop("enable_norm_bias_tuning", False)enable_quanted_input: bool = kwargs.pop("enable_quanted_input", True)disable_deterministic_algorithms = kwargs.pop("disable_deterministic_algorithms", False)- if not disable_deterministic_algorithms:- if "CUBLAS_WORKSPACE_CONFIG" not in os.environ:- os.environ["CUBLAS_WORKSPACE_CONFIG"] = ":4096:8"- torch.use_deterministic_algorithms(True, warn_only=False)-+ static_kv_dtype = kwargs.pop("static_kv_dtype", None)self.vlm = kwargs.pop("vlm") if "vlm" in kwargs else False+if kwargs:- logger.warning(f"unrecognized keys {list(kwargs.keys())} were passed. Please check them.")- self.quantized = False- self.seed = seed- set_seed(self.seed)+ logger.warning(f"Unrecognized keys {list(kwargs.keys())} were passed. Please check them.")+if device is not None and "," in str(device):raise ValueError("API does not support explicit set multiple devices,"" please set CUDA_VISIBLE_DEVICES yourself and use `device=auto` instead")- if isinstance(model, str):- model, tokenizer, low_cpu_mem_usage = llm_load_model(- model, device=device, low_cpu_mem_mode=low_cpu_mem_usage- )- elif tokenizer is None and iters > 0:- raise ValueError("A tokenizer must be set for non-str model input")- self.low_cpu_mem_usage = bool(low_cpu_mem_usage)- if unsupport_meta_device(model):- raise RuntimeError(- "AutoRound does not support parameters on meta device. "- "Please use more GPUs by setting `--device 0,1,2,3` or just place the model on CPU."- )- model = _handle_moe_model(model)- # self.model_orig_dtype = model.dtype- self.enable_alg_ext = enable_alg_ext- self.device = detect_device(device) ##must place after llm_load_model, because this one will convert auto+ if not disable_deterministic_algorithms:+ if "CUBLAS_WORKSPACE_CONFIG" not in os.environ:+ os.environ["CUBLAS_WORKSPACE_CONFIG"] = ":4096:8"+ torch.use_deterministic_algorithms(True, warn_only=False)- ## important tuning hype-parameters- self.amp = amp- self.enable_quanted_input = enable_quanted_input- self.enable_minmax_tuning = enable_minmax_tuning- self.nsamples = nsamples+ # Set quantization configsself.bits = bits- self.enable_norm_bias_tuning = enable_norm_bias_tuning- self.group_size = group_size- self.sym = sym-- self.low_gpu_mem_usage = low_gpu_mem_usage-- self.layer_config = {} if layer_config is None else layer_config- self.seqlen = seqlen- self.batch_size, self.gradient_accumulate_steps = batch_size, gradient_accumulate_steps- self.nblocks = nblocks- self.dataset = dataset- self.iters = iters- if self.iters < 0:- logger.warning("`iters` must be non-negative, reset it to 200")- self.iters = 200- if self.iters == 0:- self.lr = 5e-3- else:- self.lr = lr or (1.0 / self.iters) ##must after iter setting- self.minmax_lr = minmax_lr or self.lr-self.data_type = data_typetmp_bits = infer_bits_by_data_type(self.data_type)if tmp_bits < 16 and tmp_bits != bits:logger.warning(- f"bits set in 'data_type' do not match the specified 'bits' setting. Resetting 'bits' to {tmp_bits}."+ f"Bits set in 'data_type' do not match the specified 'bits' setting. Resetting 'bits' to {tmp_bits}.")self.bits = tmp_bits- self.supported_types = SUPPORTED_LAYER_TYPES- self.inner_supported_types = INNER_SUPPORTED_LAYER_TYPES- if "hpu" in str(self.device):- self.inner_supported_types = tuple(x for x in INNER_SUPPORTED_LAYER_TYPES if x != "FP8Linear")-- self.model = model.eval()- self.tokenizer = tokenizer-- self.scale_dtype = convert_dtype_str2torch(scale_dtype)- self.set_amp_dtype()+ self.group_size = group_size+ self.sym = sym+ self.layer_config = {} if layer_config is None else layer_configself.to_quant_block_names = to_quant_block_namesif not hasattr(self, "quant_block_list"):all_blocks = get_block_names(model)self.quant_block_list = find_matching_blocks(model, all_blocks, self.to_quant_block_names)- self.cache_device = torch.device("cpu") if self.low_gpu_mem_usage else self.device- ##activation+ # Activationself.act_group_size = act_group_size if act_group_size is not None else group_sizeself.act_bits = act_bits if act_bits is not None else self.bitsself.act_sym = act_sym if act_sym is not None else self.sym@@ -289,86 +249,106 @@ class AutoRound(object):if self.act_data_type is None:if data_type in SUPPORTED_DTYPES and self.act_bits < 16:self.act_data_type = data_type- logger.info(f"activation adopts {data_type}")+ logger.info(f"Activation adopts {data_type}")else:self.act_data_type = "float"-tmp_act_bits = infer_bits_by_data_type(self.act_data_type)if tmp_act_bits < 16 and tmp_act_bits != self.act_bits:self.act_bits = tmp_act_bitslogger.warning(- f"act_bits set in 'act_data_type' do not"+ f" 'act_data_type' do not"f" match the specified 'act_bits' setting. Resetting 'act_bits' to {tmp_act_bits}.")- # kv cache- static_kv_dtype = kwargs.pop("static_kv_dtype", None)- self.static_kv_dtype = static_kv_dtype- if self.static_kv_dtype is not None:- logger.warning("The static kv is experimental and currently has limited support.")-+ # Tuning hyperparameters+ self.seed = seed+ set_seed(self.seed)+ self.amp = amp+ self.enable_quanted_input = enable_quanted_input+ self.enable_minmax_tuning = enable_minmax_tuning+ self.nsamples = nsamples+ self.enable_norm_bias_tuning = enable_norm_bias_tuning+ self.low_gpu_mem_usage = low_gpu_mem_usage+ self.seqlen = seqlen+ self.batch_size, self.gradient_accumulate_steps = batch_size, gradient_accumulate_steps+ self.nblocks = nblocks+ self.dataset = dataset+ self.iters = iters+ if self.iters < 0:+ logger.warning("`iters` must be non-negative, reset it to 200")+ self.iters = 200+ if self.iters == 0:+ self.lr = 5e-3+ else:+ self.lr = lr or (1.0 / self.iters) # must place after iter setting+ self.minmax_lr = minmax_lr or self.lr+ self.enable_alg_ext = enable_alg_extself.sampler = samplerself.not_use_best_mse = not_use_best_mseself.dynamic_max_gap = dynamic_max_gapself.lr_scheduler = lr_schedulerself.optimizer = self.get_optimizer(None)- self.batch_dim = None- self.infer_bs_coeff = 1-self.super_bits = super_bitsself.super_group_size = super_group_size-self.disable_opt_rtn = disable_opt_rtn+ self.is_packing_immediate = False # whether to pack the layer immediately after tuning- torch.set_printoptions(precision=3, sci_mode=True)- self.check_configs()+ # KV cache, this one does not affect tuning but will collect some infos during tuning+ self.static_kv_dtype = static_kv_dtype+ if self.static_kv_dtype is not None:+ logger.warning("The static kv is experimental and currently has limited support.")++ # Model related+ self.quantized = False+ if isinstance(model, str):+ model, tokenizer, low_cpu_mem_usage = llm_load_model(+ model, device=device, low_cpu_mem_mode=low_cpu_mem_usage+ )+ elif tokenizer is None and iters > 0:+ raise ValueError("A tokenizer must be set for non-str model input")+ self.low_cpu_mem_usage = bool(low_cpu_mem_usage)+ if unsupport_meta_device(model):+ raise RuntimeError(+ "AutoRound does not support parameters on meta device. "+ "Please use more GPUs by setting `--device 0,1,2,3` or just place the model on CPU."+ )+ model = _handle_moe_model(model)+ self.model = model.eval()+ self.tokenizer = tokenizer+ self.shared_cache_keys = get_shared_keys(self.model)++ # Set device, must place after model loading+ self.device = detect_device(device) # must place after llm_load_model, because this one will convert auto+ self.device_map = device_map+ self._set_device_map_in_blocks(self.device_map)+ self.scale_dtype = convert_dtype_str2torch(scale_dtype)+ self.set_amp_dtype()+ self.cache_device = torch.device("cpu") if self.low_gpu_mem_usage else self.deviceif self.act_bits <= 8 and self.amp_dtype == torch.float16:- logger.warning("force to use bf16 to for quantization tuning when enabling activation quantization")+ logger.warning("Force to use bf16 to for quantization tuning when enabling activation quantization")self.amp_dtype = torch.bfloat16self.model = self.model.to(torch.bfloat16)else:logger.info(f"using {self.model.dtype} for quantization tuning")+ # Some helpers+ self.supported_types = SUPPORTED_LAYER_TYPES+ self.inner_supported_types = INNER_SUPPORTED_LAYER_TYPES+ if "hpu" in str(self.device):+ self.inner_supported_types = tuple(x for x in INNER_SUPPORTED_LAYER_TYPES if x != "FP8Linear")+ self.batch_dim = None+ self.infer_bs_coeff = 1self.enable_torch_compile = enable_torch_compile- if (- not self.enable_torch_compile- and TORCH_VERSION_AT_LEAST_2_6- and self.act_bits > 8- and not is_debug_mode()- and not self.low_cpu_mem_usage- and "fp8" not in self.data_type- and "fp8" not in self.act_data_type- ):- logger.info(- "'enable_torch_compile' is set to `False` by default. "- "Enabling it can reduce tuning cost by 20%, but it might throw an exception."- )-- if self.act_bits <= 8 and self.enable_torch_compile:- self.enable_torch_compile = False- logger.warning("reset enable_torch_compile to `False` as activation quantization is enabled")-- if self.low_cpu_mem_usage and self.enable_torch_compile:- self.enable_torch_compile = False- logger.warning("reset enable_torch_compile to `False` as low_cpu_mem_usage is enabled")+ self._adjust_torch_compile(enable_torch_compile)- if is_debug_mode() and self.enable_torch_compile:- self.enable_torch_compile = False- logger.warning("reset enable_torch_compile to `False` as debug mode is enabled")+ self._check_configs()- if ("fp8" in self.data_type or "fp8" in self.act_data_type) and self.enable_torch_compile:- self.enable_torch_compile = False- logger.warning("reset enable_torch_compile to `False` as fp8 is enabled")+ torch.set_printoptions(precision=3, sci_mode=True)if is_optimum_habana_available():logger.info("Optimum Habana is available, import htcore explicitly.")import habana_frameworks.torch.core as htcore # pylint: disable=E0401import habana_frameworks.torch.hpu as hthpu # pylint: disable=E0401]- self.device_map = device_map-- self.set_device_map_in_blocks(self.device_map)-- self.is_packing_immediate = False ## whether to pack the layer immediately after tuningself.serialization_keys = ["bits",@@ -398,13 +378,44 @@ class AutoRound(object):"super_group_size",]- self.shared_cache_keys = get_shared_keys(self.model)+ def _adjust_torch_compile(self, enable_torch_compile: bool) -> None:+ """Sets the torch compile configuration for the tuning."""+ self.enable_torch_compile = enable_torch_compile+ if (+ not self.enable_torch_compile+ and TORCH_VERSION_AT_LEAST_2_6+ and self.act_bits > 8+ and not is_debug_mode()+ and not self.low_cpu_mem_usage+ and "fp8" not in self.data_type+ and "fp8" not in self.act_data_type+ ):+ logger.info(+ "'enable_torch_compile' is set to `False` by default. "+ "Enabling it can reduce tuning cost by 20%, but it might throw an exception."+ )- def set_device_map_in_blocks(self, device_map):+ if self.act_bits <= 8 and self.enable_torch_compile:+ self.enable_torch_compile = False+ logger.warning("reset enable_torch_compile to `False` as activation quantization is enabled")++ if self.low_cpu_mem_usage and self.enable_torch_compile:+ self.enable_torch_compile = False+ logger.warning("reset enable_torch_compile to `False` as low_cpu_mem_usage is enabled")++ if is_debug_mode() and self.enable_torch_compile:+ self.enable_torch_compile = False+ logger.warning("reset enable_torch_compile to `False` as debug mode is enabled")++ if ("fp8" in self.data_type or "fp8" in self.act_data_type) and self.enable_torch_compile:+ self.enable_torch_compile = False+ logger.warning("reset enable_torch_compile to `False` as fp8 is enabled")++ def _set_device_map_in_blocks(self, device_map: Union[str, dict, None]) -> None:"""Sets the device map for specific blocks in the model.Args:- device_map (Union[str, dict]): A mapping of module names to devices.+ device_map (Union[str, dict, None]): A mapping of module names to devices.If provided as a string, it should be in the format"module_name:device,module_name:device". Devices can be integers(GPU IDs) or strings (e.g., 'cpu', 'cuda:0').@@ -443,7 +454,8 @@ class AutoRound(object):if key in name:self._set_device_for_matching_module(name, device)- def _set_device_for_matching_module(self, name, device):+ def _set_device_for_matching_module(self, name: str, device: str) -> None:+ """Sets the device for a module if it matches the given name."""module = get_module(self.model, name)if hasattr(module, "tuning_device") and module.tuning_device != device:logger.warning(@@ -452,7 +464,7 @@ class AutoRound(object):else:module.tuning_device = device- def _dq_check(self):+ def _dq_check(self) -> None:"""Reset the default value of super_bits and super_group_size"""if self.data_type.endswith("_dq"):gguf_config = GGUF_INNER_CONFIG[f"gguf:q{self.bits}_k"]@@ -461,56 +473,56 @@ class AutoRound(object):gguf_config["super_group_size"] if self.super_group_size is None else self.super_group_size)- def check_configs(self):+ def _check_configs(self) -> None:"""Checks if the configurations are valid.Raises:ValueError, TypeError: If any of the configurations are invalid."""if not isinstance(self.model, torch.nn.Module):- raise TypeError("model must be an instance of torch.nn.Module")+ raise TypeError("Model must be an instance of torch.nn.Module")if self.bits <= 0:- raise ValueError("bits must be positive")+ raise ValueError("`bits` must be positive")if self.act_bits <= 0:- raise ValueError("act_bits must be positive")+ raise ValueError("`act_bits` must be positive")if not (self.group_size == -1 or self.group_size >= 0):- raise ValueError("group_size must be -1 (per channel) or 0 (per-tensor) or a postivie integer")+ raise ValueError("`group_size` must be -1 (per channel) or 0 (per-tensor) or a positive integer")if not (self.act_group_size == -1 or self.act_group_size >= 0):- raise ValueError("act_group_size must be -1 (per channel) or 0 (per-tensor) or a positive integer")+ raise ValueError("`act_group_size` must be -1 (per channel) or 0 (per-tensor) or a positive integer")if self.batch_size <= 0:- raise ValueError("batch_size must be positive")+ raise ValueError("`batch_size` must be positive")if self.iters < 0:- raise ValueError("iters must be non-negative")+ raise ValueError("`iters` must be non-negative")if self.seqlen <= 0:- raise ValueError("seqlen must be positive")+ raise ValueError("`seqlen` must be positive")if self.nblocks <= 0:- raise ValueError("nblocks must be positive")+ raise ValueError("`nblocks` must be positive")if self.gradient_accumulate_steps <= 0:- raise ValueError("gradient_accumulate_steps must be positive")- # assert self.tokenizer != None or self.dataloader != None+ raise ValueError("`gradient_accumulate_steps` must be positive")+if self.act_bits <= 8:logger.warning(- "activation quantization is an experimental feature with limited support and a complex API. "+ "Activation quantization is an experimental feature with limited support and a complex API. ""And please save the quantized model to fake format as real deployment is not supported currently")if is_mx_fp(self.data_type) and self.group_size != 32:- logger.warning("mx_fp should only support group_size of 32 in real deployment")+ logger.warning("Dtype mx_fp should only support group_size of 32 in real deployment")if is_nv_fp(self.data_type) and (self.group_size != 16):- logger.warning("nv_fp should only support group_size of 16 in real deployment")+ logger.warning("Dtype nv_fp should only support group_size of 16 in real deployment")if self.nsamples < self.gradient_accumulate_steps * self.batch_size:if self.batch_size > self.nsamples:logger.warning(- f"reset batch_size to {self.nsamples} as nsamples({self.nsamples})"+ f"Reset `batch_size` to {self.nsamples} as `nsamples`({self.nsamples})"f" is smaller than batch_size({self.batch_size})")self.batch_size = self.nsamplesif self.gradient_accumulate_steps > self.nsamples // self.batch_size:self.gradient_accumulate_steps = self.nsamples // self.batch_sizelogger.warning(- f"reset gradient_accumulate_steps to {self.gradient_accumulate_steps}"+ f"Reset `gradient_accumulate_steps` to {self.gradient_accumulate_steps}"f" as nsamples must equal or greater"f" than gradient_accumulate_steps * batch_size")@@ -520,8 +532,9 @@ class AutoRound(object):self._dq_check()- def _check_compatibility(self):- ##check gguf and others+ def _check_compatibility(self) -> None:+ """Checks compatibility of the configurations and model."""+ # Check gguf and othershas_gguf = Falseif hasattr(self, "formats"):has_besides_gguf = False@@ -531,7 +544,7 @@ class AutoRound(object):elif format_ != "fake":has_besides_gguf = Trueif has_gguf and has_besides_gguf:- raise ValueError("gguf format is not compatible with other formats, please choose only one of them")+ raise ValueError("Gguf format is not compatible with other formats, please choose only one of them")if has_gguf and self.iters != 0 and self.bits != 3:logger.warning("`iters=0` is recommended when exporting to GGUF format except for bits 3,"@@ -539,7 +552,7 @@ class AutoRound(object):" We are likely to release new algorithm for certain configurations in the future.")- ##check group_size 32 for auto_round+ # Check group_size 32 for auto_roundif (self.data_type == "int"and hasattr(self, "formats")@@ -561,7 +574,7 @@ class AutoRound(object):):if self.model.config.max_position_embeddings < self.seqlen:logger.warning(- f"change sequence length to {self.model.config.max_position_embeddings} "+ f"Change sequence length to {self.model.config.max_position_embeddings} ""due to the limitation of max_position_embeddings")self.seqlen = min(self.seqlen, self.model.config.max_position_embeddings)@@ -569,16 +582,16 @@ class AutoRound(object):if self.seqlen is not None and hasattr(self.tokenizer, "model_max_length"):if self.tokenizer.model_max_length < self.seqlen:logger.warning(- f"change sequence length to {self.tokenizer.model_max_length} "+ f"Change sequence length to {self.tokenizer.model_max_length} ""due to the limitation of model_max_length. ""You can also try to increase the model_max_length to avoid this issue.")self.seqlen = min(self.seqlen, self.tokenizer.model_max_length)if self.group_size == 0 and "fp8" not in self.data_type:- logger.warning("group_size of 0 is not supported for data_type other than fp8 ")+ logger.warning("`group_size==0` is not supported for data_type other than fp8 ")- def parse_format_to_list(self, format: str) -> list:+ def _parse_format_to_list(self, format: str) -> list:"""Parses the format string into a list of formats.This method checks the requested format(s) against the model's@@ -670,7 +683,7 @@ class AutoRound(object):if format == "fake":return formatformat = format.replace("q*_", f"q{self.bits}_")- # only support to export afp8/nv_fp+ # Only support to export afp8/nv_fpif self.act_bits <= 8:if not is_standard_fp(self.act_data_type) or self.act_dynamic:if format == "llmcompressor":@@ -743,7 +756,7 @@ class AutoRound(object):self.orig_output_dir = output_dir# check and update the format based on the current configuration- format_list = self.parse_format_to_list(format)+ format_list = self._parse_format_to_list(format)self.formats = format_list# If multiple formats are specified, enforce inplace=False@@ -2695,7 +2708,7 @@ class AutoRound(object):if not self.quantized:logger.warning("please run autoround.quantize first")return- if format == "fake" or format == "qdq": ##TODO fix act quantizaiton later+ if format == "fake" or format == "qdq": # TODO fix act quantization laterself.model = self.model.to("cpu")self.model.save_pretrained(output_dir)if self.tokenizer is not None:@@ -2905,7 +2918,7 @@ class AutoRound(object):return current_input_ids, current_input_others-class AutoRoundOPT(AutoRound):+class AutoRoundAdam(AutoRound):"""Class for automatic rounding-based quantization with optimizers like adamw of a PyTorch model.Args:@@ -2999,7 +3012,7 @@ class AutoRoundOPT(AutoRound):disable_opt_rtn: bool = False,**kwargs,):- super(AutoRoundOPT, self).__init__(+ super(AutoRoundAdam, self).__init__(model=model,tokenizer=tokenizer,bits=bits,@@ -3039,6 +3052,7 @@ class AutoRoundOPT(AutoRound):device_map=device_map,super_bits=super_bits,super_group_size=super_group_size,+ disable_opt_rtn=disable_opt_rtn,**kwargs,)@@ -3082,142 +3096,3 @@ class AutoRoundOPT(AutoRound):lr_schedule.step()if is_optimum_habana_available():htcore.mark_step()---class AutoRoundAdam(AutoRoundOPT):- """Class for automatic rounding-based quantization with optimizers like adamw of a PyTorch model.- The default lr has been changed.-- Args:- model: The PyTorch model to be quantized.- tokenizer: An optional tokenizer for processing input data.- bits (int): Number of bits for quantization (default is 4).- group_size (int): Size of the quantization group (default is 128).- sym (str): Whether symmetric quantization to be used (default is True).- layer_config (dict): Configuration for weight quantization (default is None).- batch_size (int): Batch size for training (default is 8).- amp (bool): Whether to use automatic mixed precision (default is True).- device: The device to be used for training (default is "auto").- lr_scheduler: The learning rate scheduler to be used.- dataset (Union[str, list, tuple, torch.utils.data.DataLoader]):- The default dataset name (default is "NeelNanda/pile-10k").- enable_quanted_input (bool): Whether to use quantized input data (default is True).- enable_minmax_tuning (bool): Whether to enable min-max tuning (default is True).- lr (float): The learning rate (default is 0.005).- minmax_lr (float): The learning rate for min-max tuning (default is None).- low_gpu_mem_usage (bool): Whether to use low GPU memory (default is False).- low_cpu_mem_usage (bool): Whether to use low CPU memory (default is False).- iters (int): Number of iterations (default is 200).- seqlen (int): Length of the sequence.- nsamples (int): Number of samples (default is 128).- sampler (str): The sampling method (default is "rand").- seed (int): The random seed (default is 42).- nblocks (int): Number of blocks (default is 1).- gradient_accumulate_steps (int): Number of gradient accumulation steps (default is 1).- not_use_best_mse (bool): Whether to use mean squared error (default is False).- dynamic_max_gap (int): The dynamic maximum gap (default is -1).- data_type (str): The data type to be used (default is "int").- optimizer: string or object- scale_dtype (str): The data type of quantization scale to be used (default is "float16"), different kernels- have different choices.- act_bits (int): Number of bits for activation quantization. Default is 16.- act_group_size (int): Group size for activation quantization. Default is None.- act_sym (bool): Whether to use symmetric activation quantization. Default is None.- act_data_type (str): Specifies the data type for activations.- Defaults to None, in which case it inherits the weight data type.- act_dynamic (bool): Whether to use dynamic activation quantization. Default is True.- to_quant_block_names (str|list): A list whose elements are list of block's layer names to be quantized.- enable_norm_bias_tuning (bool): Whether to enable fast norm/layer_bias tuning- enable_torch_compile (bool): Whether to enable torch compile to optimize quant_block/layer function- Returns:- The quantized model.- """-- def __init__(- self,- model: Union[torch.nn.Module, str],- tokenizer=None,- bits: int = 4,- group_size: int = 128,- sym: bool = True,- layer_config=None,- batch_size: int = 8,- amp: bool = True,- device: Union[str, torch.device, int] = 0,- lr_scheduler=None,- dataset: Union[str, list, tuple, torch.utils.data.DataLoader] = "NeelNanda/pile-10k",- enable_quanted_input: bool = True,- enable_minmax_tuning: bool = True,- lr: float = None,- minmax_lr: float = None,- low_gpu_mem_usage: bool = False,- low_cpu_mem_usage: int = 0,- iters: int = 200,- seqlen: int = 2048,- nsamples: int = 128,- sampler: str = "rand",- seed: int = 42,- nblocks: int = 1,- gradient_accumulate_steps: int = 1,- not_use_best_mse: bool = False,- dynamic_max_gap: int = -1,- data_type: str = "int",- scale_dtype: str = "fp16",- act_bits: int = 16,- act_group_size: int = None,- act_sym: bool = None,- act_data_type: str = None,- act_dynamic: bool = True,- to_quant_block_names: Union[str, list] = None,- enable_norm_bias_tuning: bool = False,- enable_torch_compile: bool = False,- device_map: Union[str, dict] = None,- optimizer="AdamW",- super_bits: int = None,- super_group_size: int = None,- disable_opt_rtn: bool = False,- **kwargs,- ):- super(AutoRoundAdam, self).__init__(- model=model,- tokenizer=tokenizer,- bits=bits,- group_size=group_size,- sym=sym,- layer_config=layer_config,- batch_size=batch_size,- amp=amp,- device=device,- lr_scheduler=lr_scheduler,- dataset=dataset,- enable_quanted_input=enable_quanted_input,- enable_minmax_tuning=enable_minmax_tuning,- lr=lr,- minmax_lr=minmax_lr,- low_gpu_mem_usage=low_gpu_mem_usage,- low_cpu_mem_usage=low_cpu_mem_usage,- iters=iters,- seqlen=seqlen,- nsamples=nsamples,- sampler=sampler,- seed=seed,- nblocks=nblocks,- gradient_accumulate_steps=gradient_accumulate_steps,- not_use_best_mse=not_use_best_mse,- dynamic_max_gap=dynamic_max_gap,- data_type=data_type,- scale_dtype=scale_dtype,- act_bits=act_bits,- act_group_size=act_group_size,- act_sym=act_sym,- act_data_type=act_data_type,- act_dynamic=act_dynamic,- to_quant_block_names=to_quant_block_names,- enable_norm_bias_tuning=enable_norm_bias_tuning,- enable_torch_compile=enable_torch_compile,- device_map=device_map,- optimizer=optimizer,- super_bits=super_bits,- super_group_size=super_group_size,- **kwargs,- )diff --git a/auto_round/inference/backend.py b/auto_round/inference/backend.pyindex a4f578726..ea66a0389 100644--- a/auto_round/inference/backend.py+++ b/auto_round/inference/backend.py@@ -115,7 +115,7 @@ BackendInfos["auto_gptq:exllamav2"] = BackendInfo(group_size=[-1, 32, 64, 128, 256, 512, 1024, 2048],feature_checks=[exllamav2_feature_check],alias=["gptq", "auto_gptq", "exllamav2", "gptq:exllamav2", "auto_gptq:exllamav2"],- requirements=["auto-gptq>=0.7.1"],+ requirements=["torch<2.6.0", "auto-gptq>=0.7.1"],)BackendInfos["auto_gptq:tritonv2"] = BackendInfo(@@ -128,7 +128,7 @@ BackendInfos["auto_gptq:tritonv2"] = BackendInfo(priority=0,feature_checks=[exllamav2_feature_check],alias=["auto_gptq:tritonv2"],- requirements=["auto-gptq>=0.7.1", "triton>=2.0"],+ requirements=["torch<2.6.0", "auto-gptq>=0.7.1", "triton>=2.0"],)BackendInfos["auto_gptq:cuda"] = BackendInfo(@@ -142,7 +142,10 @@ BackendInfos["auto_gptq:cuda"] = BackendInfo(alias=["auto_gptq:cuda"],dtype=["float16"],convertable_format=["int32_zp"],- requirements=["auto-gptq>=0.7.1"],+ requirements=[+ "torch<2.6.0",+ "auto-gptq>=0.7.1",+ ],)BackendInfos["auto_round:tritonv2"] = BackendInfo(@@ -164,7 +167,7 @@ BackendInfos["auto_round:torch"] = BackendInfo(dtype=["float16", "bfloat16"],bits=[2, 3, 4, 8],priority=0,- feature_checks=[feature_multiply_checker_32],+ feature_checks=[exllamav2_feature_check],alias=["auto_round", "torch"],requirements=["auto-round>=0.5.1"],)@@ -189,7 +192,7 @@ BackendInfos["auto_round:torch_zp"] = BackendInfo(dtype=["float16", "bfloat16"],bits=[2, 3, 4, 8],priority=0,- feature_checks=[feature_multiply_checker_32],+ feature_checks=[exllamav2_feature_check],alias=["torch", "torch_zp"],requirements=["auto-round>=0.5.1"],)@@ -256,7 +259,7 @@ BackendInfos["qbits"] = BackendInfo(alias=["itrex", "qbits"],dtype=["float16", "bfloat16"],convertable_format=["int32"],- requirements=["intel-extension-for-transformers", "torch<2.7.0"],+ requirements=["torch<2.7.0", "intel-extension-for-transformers"],)BackendInfos["qbits_zp"] = BackendInfo(@@ -270,7 +273,7 @@ BackendInfos["qbits_zp"] = BackendInfo(feature_checks=[],alias=["itrex", "qbits"],convertable_format=["int32_zp"],- requirements=["intel-extension-for-transformers", "torch<2.7.0"],+ requirements=["torch<2.7.0", "intel-extension-for-transformers"],)BackendInfos["auto_round:qbits_awq"] = BackendInfo(@@ -283,7 +286,7 @@ BackendInfos["auto_round:qbits_awq"] = BackendInfo(priority=1,feature_checks=[],alias=["itrex", "qbits"],- requirements=["intel-extension-for-transformers", "torch<2.7.0"],+ requirements=["torch<2.7.0", "intel-extension-for-transformers"],)BackendInfos["ipex_gptq"] = BackendInfo(diff --git a/auto_round_extension/triton/qlinear_tritonv2.py b/auto_round_extension/triton/qlinear_tritonv2.pyindex aabf90a1c..05b6fea15 100644--- a/auto_round_extension/triton/qlinear_tritonv2.py+++ b/auto_round_extension/triton/qlinear_tritonv2.py@@ -179,8 +179,8 @@ class QuantLinear(nn.Module, TritonModuleMixin):self.bits,self.maxq,)- out = out.half().reshape(out_shape)- out = out + self.bias if self.bias is not None else out+ out = out.to(x.dtype).reshape(out_shape)+ out = out + self.bias.to(x.dtype) if self.bias is not None else outreturn out.to(x.dtype)@classmethoddiff --git a/auto_round_extension/triton/qlinear_tritonv2_zp.py b/auto_round_extension/triton/qlinear_tritonv2_zp.pyindex 222e1d013..36ecd6601 100644--- a/auto_round_extension/triton/qlinear_tritonv2_zp.py+++ b/auto_round_extension/triton/qlinear_tritonv2_zp.py@@ -190,8 +190,8 @@ class QuantLinear(nn.Module, TritonModuleMixin):self.bits,self.maxq,)- out = out.half().reshape(out_shape)- out = out + self.bias if self.bias is not None else out+ out = out.to(x.dtype).reshape(out_shape)+ out = out + self.bias.to(x.dtype) if self.bias is not None else outreturn out.to(x.dtype)@classmethoddiff --git a/test/test_cpu/test_autoopt.py b/test/test_cpu/test_autoopt.pyindex fc6446dea..4f07c4e22 100644--- a/test/test_cpu/test_autoopt.py+++ b/test/test_cpu/test_autoopt.py@@ -8,7 +8,7 @@ import torchimport transformersfrom transformers import AutoModelForCausalLM, AutoTokenizer-from auto_round import AutoRoundAdam, AutoRoundOPT+from auto_round import AutoRoundAdamclass LLMDataLoader:@@ -33,21 +33,6 @@ class TestAutoRound(unittest.TestCase):shutil.rmtree("./saved", ignore_errors=True)shutil.rmtree("runs", ignore_errors=True)- def test_default(self):- bits, group_size, sym = 4, 128, False- autoround = AutoRoundOPT(- self.model,- self.tokenizer,- bits=bits,- group_size=group_size,- sym=sym,- iters=2,- seqlen=10,- dataset=self.llm_dataloader,- to_quant_block_names=None,- )- autoround.quantize()-def test_Adam(self):bits, group_size, sym = 4, 128, Falsefrom auto_round.utils import get_block_names