Skip to content

PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度

refine code (#749)

Wenhua Chenge9b2b83bb8d7
diff --git a/auto_round/__init__.py b/auto_round/__init__.py
index 7d5215f68..93cce6299 100644
--- a/auto_round/__init__.py
+++ b/auto_round/__init__.py
@@ -11,7 +11,7 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
-from auto_round.autoround import AutoRound, AutoRoundAdam, AutoRoundOPT
+from auto_round.autoround import AutoRound, AutoRoundAdam
from auto_round.mllm import AutoRoundMLLM
from auto_round.utils import LazyImport
diff --git a/auto_round/autoround.py b/auto_round/autoround.py
index 8faf3f9b8..20d01241d 100644
--- a/auto_round/autoround.py
+++ b/auto_round/autoround.py
@@ -83,69 +83,21 @@ from auto_round.wrapper import WrapperLinear, WrapperMultiblock, unwrapper_block
class AutoRound(object):
- """For more information, please refer to Cheng, Wenhua, et al. "Optimize weight rounding via signed gradient descent
- for the quantization of llms." arXiv preprint arXiv:2309.05516 (2023).
-
- Args:
- model: The PyTorch model to be quantized.
- tokenizer: An optional tokenizer for processing input data. If none is provided, a dataloader must be supplied.
- bits (int): Number of bits for quantization (default is 4).
- group_size (int): Size of the quantization group (default is 128).
- sym (bool): Whether symmetric quantization is to be used (default is True).
- layer_config (dict): Configuration for weight quantization (default is None).
- layer_config={
- 'layer1':##layer_name
- {
- 'data_type': 'int',
- 'bits': 4,
- 'group_size': 128,
- 'sym': True
- 'act_data_type': None,
- 'act_bits': 16,
- 'act_group_size': None,
- 'act_sym': None,
-
- }
- ...
- }
- batch_size (int): Batch size for training (default is 8).
- amp (bool): Whether to use automatic mixed precision (default is True).
- device: The device to be used for tuning (default is "auto").
- lr_scheduler: The learning rate scheduler to be used.
- dataset (str): The default dataset name (default is "NeelNanda/pile-10k").
- enable_quanted_input (bool): Whether to use the output of the previous quantized block as
- the input for the current block (default is True).
- enable_minmax_tuning (bool): Whether to enable weight min-max tuning (default is True).
- lr (float): The learning rate (default is None, will be set to 1.0/iters).
- minmax_lr (float): The learning rate for min-max tuning (default is None, it will be set to lr automatically).
- low_gpu_mem_usage (bool): Whether to use low GPU memory (default is True).
- low_cpu_mem_usage (bool): Whether to use low CPU memory (default is False).
- iters (int): Number of iterations (default is 200).
- seqlen (int): Data length of the sequence for tuning (default is 2048).
- nsamples (int): Number of samples (default is 128).
- sampler (str): The sampling method (default is "rand").
- seed (int): The random seed (default is 42).
- nblocks (int): Number of blocks (default is 1).
- gradient_accumulate_steps (int): Number of gradient accumulation steps (default is 1).
- not_use_best_mse (bool): Whether to use mean squared error (default is False).
- dynamic_max_gap (int): The dynamic maximum gap (default is -1).
- data_type (str): The data type to be used (default is "int").
- scale_dtype (str): The data type of quantization scale to be used (default is "float16"), different kernels
- have different choices.
- act_bits (int): Number of bits for activation quantization. Default is 16.
- act_group_size (int): Group size for activation quantization. Default is None.
- act_sym (bool): Whether to use symmetric activation quantization. Default is None.
- act_data_type (str): Specifies the data type for activations.
- Defaults to None, in which case it inherits the weight data type.
- act_dynamic (bool): Whether to use dynamic activation quantization. Default is True.
- to_quant_block_names (str|list): A string or list whose elements are list of
- block's layer names to be quantized.
- enable_norm_bias_tuning (bool): Whether to enable fast norm/layer_bias tuning
- enable_torch_compile (bool): Whether to enable torch compile to optimize quant_block/layer (default it False).
- device_map (str|dict): device map for each block
- disable_opt_rtn (bool): Whether to disable optimization of the RTN mode(iters=0) (default is False).
- Returns:
- The quantized model.
+ """Automatic weight rounding (Signed Gradient Descent) for LLM quantization
+
+ Reference:
+ Cheng, Wenhua, et al., "Optimize weight rounding via signed gradient descent for
+ the quantization of LLMs." arXiv:2309.05516 (2023).
+
+ Attributes:
+ model (torch.nn.Module): The loaded PyTorch model in eval mode.
+ tokenizer: Tokenizer used to prepare input text for calibration/tuning.
+ bits (int): Weight quantization bits.
+ group_size (int): Per-group size for weight quantization.
+ sym (bool): Whether to use symmetric weight quantization.
+ layer_config (dict): Per-layer quantization configuration.
+ nsamples (int): Number of calibration samples.
+ enable_torch_compile (bool): Whether to enable torch.compile for quant blocks/layers.
"""
def __init__(
@@ -181,8 +133,66 @@ class AutoRound(object):
enable_alg_ext: bool = False,
**kwargs,
):
- ## to ensure backward compatibility, move infrequently used arguments to kwargs arguments.
- ## major version releases may be pack them with extra configuration options
+ """Initialize AutoRound with quantization and tuning configuration.
+
+ Args:
+ model (torch.nn.Module | str): Model object or model name to load.
+ tokenizer: Tokenizer for text processing. Required if `model` is not a string and `iters > 0`.
+ bits (int, optional): Weight quantization bits. Defaults to 4.
+ group_size (int, optional): Weight quantization group size. Defaults to 128.
+ sym (bool, optional): Symmetric weight quantization. Defaults to True.
+ layer_config (dict, optional): Layer-wise quantization config. Defaults to None.
+ batch_size (int, optional): Calibration batch size. Defaults to 8.
+ amp (bool, optional): Use AMP for tuning. Defaults to True.
+ device (str | torch.device | int, optional): Compute device. Defaults to 0.
+ dataset (str | list | tuple | DataLoader, optional): Calibration data. Defaults to "NeelNanda/pile-10k".
+ enable_minmax_tuning (bool, optional): Enable weight min-max tuning. Defaults to True.
+ lr (float, optional): Learning rate; if None, set to 1.0 / iters except when iters==0.
+ minmax_lr (float, optional): Learning rate for min-max tuning; defaults to `lr`.
+ low_gpu_mem_usage (bool, optional): Lower GPU memory mode. Defaults to False.
+ iters (int, optional): Optimization iterations. Defaults to 200.
+ seqlen (int, optional): Calibration sequence length. Defaults to 2048.
+ nsamples (int, optional): Number of calibration samples. Defaults to 128.
+ seed (int, optional): Random seed. Defaults to 42.
+ gradient_accumulate_steps (int, optional): Gradient accumulation steps. Defaults to 1.
+ data_type (str, optional): Weight data type string, e.g., "int". Defaults to "int".
+ act_bits (int, optional): Activation quantization bits. Defaults to 16.
+ act_group_size (int, optional): Activation group size. Defaults to None.
+ act_sym (bool, optional): Symmetric activation quantization. Defaults to None.
+ act_data_type (str, optional): Activation data type; inherits weight dtype if None and act_bits < 16.
+ act_dynamic (bool, optional): Dynamic activation quantization. Defaults to True.
+ enable_torch_compile (bool, optional): Enable torch.compile for quant blocks/layers. Defaults to False.
+ device_map (str | dict, optional): Device placement map. Defaults to None.
+ disable_opt_rtn (bool, optional): Disable RTN-mode optimization (iters=0). Defaults to False.
+ enable_alg_ext (bool, optional): Enable algorithm extension (primarily for INT2). Defaults to False.
+ **kwargs: Backward compatible options:
+ - lr_scheduler, sampler, not_use_best_mse, dynamic_max_gap,
+ super_group_size, super_bits, scale_dtype ("fp16" etc.),
+ nblocks, low_cpu_mem_usage, to_quant_block_names,
+ enable_norm_bias_tuning, enable_quanted_input,
+ disable_deterministic_algorithms, vlm, static_kv_dtype
+ Raises:
+ ValueError: If invalid device is provided or tokenizer is missing for non-str model with iters > 0.
+ RuntimeError: If model parameters are on meta device.
+ Example:
+ Layer-wise configuration structure:
+
+ >>> layer_config = {
+ ... "layer1": {
+ ... "data_type": "int",
+ ... "bits": 4,
+ ... "group_size": 128,
+ ... "sym": True,
+ ... "act_data_type": None,
+ ... "act_bits": 16,
+ ... "act_group_size": None,
+ ... "act_sym": None,
+ ... },
+ ... # ...
+ ... }
+ """
+ # Extra/legacy kwargs for backward compatibility
+ # Major version releases may pack them with extra configuration options
lr_scheduler = kwargs.pop("lr_scheduler", None)
sampler = kwargs.pop("sampler", "rand")
not_use_best_mse = kwargs.pop("not_use_best_mse", False)
@@ -196,91 +206,41 @@ class AutoRound(object):
enable_norm_bias_tuning: bool = kwargs.pop("enable_norm_bias_tuning", False)
enable_quanted_input: bool = kwargs.pop("enable_quanted_input", True)
disable_deterministic_algorithms = kwargs.pop("disable_deterministic_algorithms", False)
- if not disable_deterministic_algorithms:
- if "CUBLAS_WORKSPACE_CONFIG" not in os.environ:
- os.environ["CUBLAS_WORKSPACE_CONFIG"] = ":4096:8"
- torch.use_deterministic_algorithms(True, warn_only=False)
-
+ static_kv_dtype = kwargs.pop("static_kv_dtype", None)
self.vlm = kwargs.pop("vlm") if "vlm" in kwargs else False
+
if kwargs:
- logger.warning(f"unrecognized keys {list(kwargs.keys())} were passed. Please check them.")
- self.quantized = False
- self.seed = seed
- set_seed(self.seed)
+ logger.warning(f"Unrecognized keys {list(kwargs.keys())} were passed. Please check them.")
+
if device is not None and "," in str(device):
raise ValueError(
"API does not support explicit set multiple devices,"
" please set CUDA_VISIBLE_DEVICES yourself and use `device=auto` instead"
)
- if isinstance(model, str):
- model, tokenizer, low_cpu_mem_usage = llm_load_model(
- model, device=device, low_cpu_mem_mode=low_cpu_mem_usage
- )
- elif tokenizer is None and iters > 0:
- raise ValueError("A tokenizer must be set for non-str model input")
- self.low_cpu_mem_usage = bool(low_cpu_mem_usage)
- if unsupport_meta_device(model):
- raise RuntimeError(
- "AutoRound does not support parameters on meta device. "
- "Please use more GPUs by setting `--device 0,1,2,3` or just place the model on CPU."
- )
- model = _handle_moe_model(model)
- # self.model_orig_dtype = model.dtype
- self.enable_alg_ext = enable_alg_ext
- self.device = detect_device(device) ##must place after llm_load_model, because this one will convert auto
+ if not disable_deterministic_algorithms:
+ if "CUBLAS_WORKSPACE_CONFIG" not in os.environ:
+ os.environ["CUBLAS_WORKSPACE_CONFIG"] = ":4096:8"
+ torch.use_deterministic_algorithms(True, warn_only=False)
- ## important tuning hype-parameters
- self.amp = amp
- self.enable_quanted_input = enable_quanted_input
- self.enable_minmax_tuning = enable_minmax_tuning
- self.nsamples = nsamples
+ # Set quantization configs
self.bits = bits
- self.enable_norm_bias_tuning = enable_norm_bias_tuning
- self.group_size = group_size
- self.sym = sym
-
- self.low_gpu_mem_usage = low_gpu_mem_usage
-
- self.layer_config = {} if layer_config is None else layer_config
- self.seqlen = seqlen
- self.batch_size, self.gradient_accumulate_steps = batch_size, gradient_accumulate_steps
- self.nblocks = nblocks
- self.dataset = dataset
- self.iters = iters
- if self.iters < 0:
- logger.warning("`iters` must be non-negative, reset it to 200")
- self.iters = 200
- if self.iters == 0:
- self.lr = 5e-3
- else:
- self.lr = lr or (1.0 / self.iters) ##must after iter setting
- self.minmax_lr = minmax_lr or self.lr
-
self.data_type = data_type
tmp_bits = infer_bits_by_data_type(self.data_type)
if tmp_bits < 16 and tmp_bits != bits:
logger.warning(
- f"bits set in 'data_type' do not match the specified 'bits' setting. Resetting 'bits' to {tmp_bits}."
+ f"Bits set in 'data_type' do not match the specified 'bits' setting. Resetting 'bits' to {tmp_bits}."
)
self.bits = tmp_bits
- self.supported_types = SUPPORTED_LAYER_TYPES
- self.inner_supported_types = INNER_SUPPORTED_LAYER_TYPES
- if "hpu" in str(self.device):
- self.inner_supported_types = tuple(x for x in INNER_SUPPORTED_LAYER_TYPES if x != "FP8Linear")
-
- self.model = model.eval()
- self.tokenizer = tokenizer
-
- self.scale_dtype = convert_dtype_str2torch(scale_dtype)
- self.set_amp_dtype()
+ self.group_size = group_size
+ self.sym = sym
+ self.layer_config = {} if layer_config is None else layer_config
self.to_quant_block_names = to_quant_block_names
if not hasattr(self, "quant_block_list"):
all_blocks = get_block_names(model)
self.quant_block_list = find_matching_blocks(model, all_blocks, self.to_quant_block_names)
- self.cache_device = torch.device("cpu") if self.low_gpu_mem_usage else self.device
- ##activation
+ # Activation
self.act_group_size = act_group_size if act_group_size is not None else group_size
self.act_bits = act_bits if act_bits is not None else self.bits
self.act_sym = act_sym if act_sym is not None else self.sym
@@ -289,86 +249,106 @@ class AutoRound(object):
if self.act_data_type is None:
if data_type in SUPPORTED_DTYPES and self.act_bits < 16:
self.act_data_type = data_type
- logger.info(f"activation adopts {data_type}")
+ logger.info(f"Activation adopts {data_type}")
else:
self.act_data_type = "float"
-
tmp_act_bits = infer_bits_by_data_type(self.act_data_type)
if tmp_act_bits < 16 and tmp_act_bits != self.act_bits:
self.act_bits = tmp_act_bits
logger.warning(
- f"act_bits set in 'act_data_type' do not"
+ f" 'act_data_type' do not"
f" match the specified 'act_bits' setting. Resetting 'act_bits' to {tmp_act_bits}."
)
- # kv cache
- static_kv_dtype = kwargs.pop("static_kv_dtype", None)
- self.static_kv_dtype = static_kv_dtype
- if self.static_kv_dtype is not None:
- logger.warning("The static kv is experimental and currently has limited support.")
-
+ # Tuning hyperparameters
+ self.seed = seed
+ set_seed(self.seed)
+ self.amp = amp
+ self.enable_quanted_input = enable_quanted_input
+ self.enable_minmax_tuning = enable_minmax_tuning
+ self.nsamples = nsamples
+ self.enable_norm_bias_tuning = enable_norm_bias_tuning
+ self.low_gpu_mem_usage = low_gpu_mem_usage
+ self.seqlen = seqlen
+ self.batch_size, self.gradient_accumulate_steps = batch_size, gradient_accumulate_steps
+ self.nblocks = nblocks
+ self.dataset = dataset
+ self.iters = iters
+ if self.iters < 0:
+ logger.warning("`iters` must be non-negative, reset it to 200")
+ self.iters = 200
+ if self.iters == 0:
+ self.lr = 5e-3
+ else:
+ self.lr = lr or (1.0 / self.iters) # must place after iter setting
+ self.minmax_lr = minmax_lr or self.lr
+ self.enable_alg_ext = enable_alg_ext
self.sampler = sampler
self.not_use_best_mse = not_use_best_mse
self.dynamic_max_gap = dynamic_max_gap
self.lr_scheduler = lr_scheduler
self.optimizer = self.get_optimizer(None)
- self.batch_dim = None
- self.infer_bs_coeff = 1
-
self.super_bits = super_bits
self.super_group_size = super_group_size
-
self.disable_opt_rtn = disable_opt_rtn
+ self.is_packing_immediate = False # whether to pack the layer immediately after tuning
- torch.set_printoptions(precision=3, sci_mode=True)
- self.check_configs()
+ # KV cache, this one does not affect tuning but will collect some infos during tuning
+ self.static_kv_dtype = static_kv_dtype
+ if self.static_kv_dtype is not None:
+ logger.warning("The static kv is experimental and currently has limited support.")
+
+ # Model related
+ self.quantized = False
+ if isinstance(model, str):
+ model, tokenizer, low_cpu_mem_usage = llm_load_model(
+ model, device=device, low_cpu_mem_mode=low_cpu_mem_usage
+ )
+ elif tokenizer is None and iters > 0:
+ raise ValueError("A tokenizer must be set for non-str model input")
+ self.low_cpu_mem_usage = bool(low_cpu_mem_usage)
+ if unsupport_meta_device(model):
+ raise RuntimeError(
+ "AutoRound does not support parameters on meta device. "
+ "Please use more GPUs by setting `--device 0,1,2,3` or just place the model on CPU."
+ )
+ model = _handle_moe_model(model)
+ self.model = model.eval()
+ self.tokenizer = tokenizer
+ self.shared_cache_keys = get_shared_keys(self.model)
+
+ # Set device, must place after model loading
+ self.device = detect_device(device) # must place after llm_load_model, because this one will convert auto
+ self.device_map = device_map
+ self._set_device_map_in_blocks(self.device_map)
+ self.scale_dtype = convert_dtype_str2torch(scale_dtype)
+ self.set_amp_dtype()
+ self.cache_device = torch.device("cpu") if self.low_gpu_mem_usage else self.device
if self.act_bits <= 8 and self.amp_dtype == torch.float16:
- logger.warning("force to use bf16 to for quantization tuning when enabling activation quantization")
+ logger.warning("Force to use bf16 to for quantization tuning when enabling activation quantization")
self.amp_dtype = torch.bfloat16
self.model = self.model.to(torch.bfloat16)
else:
logger.info(f"using {self.model.dtype} for quantization tuning")
+ # Some helpers
+ self.supported_types = SUPPORTED_LAYER_TYPES
+ self.inner_supported_types = INNER_SUPPORTED_LAYER_TYPES
+ if "hpu" in str(self.device):
+ self.inner_supported_types = tuple(x for x in INNER_SUPPORTED_LAYER_TYPES if x != "FP8Linear")
+ self.batch_dim = None
+ self.infer_bs_coeff = 1
self.enable_torch_compile = enable_torch_compile
- if (
- not self.enable_torch_compile
- and TORCH_VERSION_AT_LEAST_2_6
- and self.act_bits > 8
- and not is_debug_mode()
- and not self.low_cpu_mem_usage
- and "fp8" not in self.data_type
- and "fp8" not in self.act_data_type
- ):
- logger.info(
- "'enable_torch_compile' is set to `False` by default. "
- "Enabling it can reduce tuning cost by 20%, but it might throw an exception."
- )
-
- if self.act_bits <= 8 and self.enable_torch_compile:
- self.enable_torch_compile = False
- logger.warning("reset enable_torch_compile to `False` as activation quantization is enabled")
-
- if self.low_cpu_mem_usage and self.enable_torch_compile:
- self.enable_torch_compile = False
- logger.warning("reset enable_torch_compile to `False` as low_cpu_mem_usage is enabled")
+ self._adjust_torch_compile(enable_torch_compile)
- if is_debug_mode() and self.enable_torch_compile:
- self.enable_torch_compile = False
- logger.warning("reset enable_torch_compile to `False` as debug mode is enabled")
+ self._check_configs()
- if ("fp8" in self.data_type or "fp8" in self.act_data_type) and self.enable_torch_compile:
- self.enable_torch_compile = False
- logger.warning("reset enable_torch_compile to `False` as fp8 is enabled")
+ torch.set_printoptions(precision=3, sci_mode=True)
if is_optimum_habana_available():
logger.info("Optimum Habana is available, import htcore explicitly.")
import habana_frameworks.torch.core as htcore # pylint: disable=E0401
import habana_frameworks.torch.hpu as hthpu # pylint: disable=E0401]
- self.device_map = device_map
-
- self.set_device_map_in_blocks(self.device_map)
-
- self.is_packing_immediate = False ## whether to pack the layer immediately after tuning
self.serialization_keys = [
"bits",
@@ -398,13 +378,44 @@ class AutoRound(object):
"super_group_size",
]
- self.shared_cache_keys = get_shared_keys(self.model)
+ def _adjust_torch_compile(self, enable_torch_compile: bool) -> None:
+ """Sets the torch compile configuration for the tuning."""
+ self.enable_torch_compile = enable_torch_compile
+ if (
+ not self.enable_torch_compile
+ and TORCH_VERSION_AT_LEAST_2_6
+ and self.act_bits > 8
+ and not is_debug_mode()
+ and not self.low_cpu_mem_usage
+ and "fp8" not in self.data_type
+ and "fp8" not in self.act_data_type
+ ):
+ logger.info(
+ "'enable_torch_compile' is set to `False` by default. "
+ "Enabling it can reduce tuning cost by 20%, but it might throw an exception."
+ )
- def set_device_map_in_blocks(self, device_map):
+ if self.act_bits <= 8 and self.enable_torch_compile:
+ self.enable_torch_compile = False
+ logger.warning("reset enable_torch_compile to `False` as activation quantization is enabled")
+
+ if self.low_cpu_mem_usage and self.enable_torch_compile:
+ self.enable_torch_compile = False
+ logger.warning("reset enable_torch_compile to `False` as low_cpu_mem_usage is enabled")
+
+ if is_debug_mode() and self.enable_torch_compile:
+ self.enable_torch_compile = False
+ logger.warning("reset enable_torch_compile to `False` as debug mode is enabled")
+
+ if ("fp8" in self.data_type or "fp8" in self.act_data_type) and self.enable_torch_compile:
+ self.enable_torch_compile = False
+ logger.warning("reset enable_torch_compile to `False` as fp8 is enabled")
+
+ def _set_device_map_in_blocks(self, device_map: Union[str, dict, None]) -> None:
"""Sets the device map for specific blocks in the model.
Args:
- device_map (Union[str, dict]): A mapping of module names to devices.
+ device_map (Union[str, dict, None]): A mapping of module names to devices.
If provided as a string, it should be in the format
"module_name:device,module_name:device". Devices can be integers
(GPU IDs) or strings (e.g., 'cpu', 'cuda:0').
@@ -443,7 +454,8 @@ class AutoRound(object):
if key in name:
self._set_device_for_matching_module(name, device)
- def _set_device_for_matching_module(self, name, device):
+ def _set_device_for_matching_module(self, name: str, device: str) -> None:
+ """Sets the device for a module if it matches the given name."""
module = get_module(self.model, name)
if hasattr(module, "tuning_device") and module.tuning_device != device:
logger.warning(
@@ -452,7 +464,7 @@ class AutoRound(object):
else:
module.tuning_device = device
- def _dq_check(self):
+ def _dq_check(self) -> None:
"""Reset the default value of super_bits and super_group_size"""
if self.data_type.endswith("_dq"):
gguf_config = GGUF_INNER_CONFIG[f"gguf:q{self.bits}_k"]
@@ -461,56 +473,56 @@ class AutoRound(object):
gguf_config["super_group_size"] if self.super_group_size is None else self.super_group_size
)
- def check_configs(self):
+ def _check_configs(self) -> None:
"""Checks if the configurations are valid.
Raises:
ValueError, TypeError: If any of the configurations are invalid.
"""
if not isinstance(self.model, torch.nn.Module):
- raise TypeError("model must be an instance of torch.nn.Module")
+ raise TypeError("Model must be an instance of torch.nn.Module")
if self.bits <= 0:
- raise ValueError("bits must be positive")
+ raise ValueError("`bits` must be positive")
if self.act_bits <= 0:
- raise ValueError("act_bits must be positive")
+ raise ValueError("`act_bits` must be positive")
if not (self.group_size == -1 or self.group_size >= 0):
- raise ValueError("group_size must be -1 (per channel) or 0 (per-tensor) or a postivie integer")
+ raise ValueError("`group_size` must be -1 (per channel) or 0 (per-tensor) or a positive integer")
if not (self.act_group_size == -1 or self.act_group_size >= 0):
- raise ValueError("act_group_size must be -1 (per channel) or 0 (per-tensor) or a positive integer")
+ raise ValueError("`act_group_size` must be -1 (per channel) or 0 (per-tensor) or a positive integer")
if self.batch_size <= 0:
- raise ValueError("batch_size must be positive")
+ raise ValueError("`batch_size` must be positive")
if self.iters < 0:
- raise ValueError("iters must be non-negative")
+ raise ValueError("`iters` must be non-negative")
if self.seqlen <= 0:
- raise ValueError("seqlen must be positive")
+ raise ValueError("`seqlen` must be positive")
if self.nblocks <= 0:
- raise ValueError("nblocks must be positive")
+ raise ValueError("`nblocks` must be positive")
if self.gradient_accumulate_steps <= 0:
- raise ValueError("gradient_accumulate_steps must be positive")
- # assert self.tokenizer != None or self.dataloader != None
+ raise ValueError("`gradient_accumulate_steps` must be positive")
+
if self.act_bits <= 8:
logger.warning(
- "activation quantization is an experimental feature with limited support and a complex API. "
+ "Activation quantization is an experimental feature with limited support and a complex API. "
"And please save the quantized model to fake format as real deployment is not supported currently"
)
if is_mx_fp(self.data_type) and self.group_size != 32:
- logger.warning("mx_fp should only support group_size of 32 in real deployment")
+ logger.warning("Dtype mx_fp should only support group_size of 32 in real deployment")
if is_nv_fp(self.data_type) and (self.group_size != 16):
- logger.warning("nv_fp should only support group_size of 16 in real deployment")
+ logger.warning("Dtype nv_fp should only support group_size of 16 in real deployment")
if self.nsamples < self.gradient_accumulate_steps * self.batch_size:
if self.batch_size > self.nsamples:
logger.warning(
- f"reset batch_size to {self.nsamples} as nsamples({self.nsamples})"
+ f"Reset `batch_size` to {self.nsamples} as `nsamples`({self.nsamples})"
f" is smaller than batch_size({self.batch_size})"
)
self.batch_size = self.nsamples
if self.gradient_accumulate_steps > self.nsamples // self.batch_size:
self.gradient_accumulate_steps = self.nsamples // self.batch_size
logger.warning(
- f"reset gradient_accumulate_steps to {self.gradient_accumulate_steps}"
+ f"Reset `gradient_accumulate_steps` to {self.gradient_accumulate_steps}"
f" as nsamples must equal or greater"
f" than gradient_accumulate_steps * batch_size"
)
@@ -520,8 +532,9 @@ class AutoRound(object):
self._dq_check()
- def _check_compatibility(self):
- ##check gguf and others
+ def _check_compatibility(self) -> None:
+ """Checks compatibility of the configurations and model."""
+ # Check gguf and others
has_gguf = False
if hasattr(self, "formats"):
has_besides_gguf = False
@@ -531,7 +544,7 @@ class AutoRound(object):
elif format_ != "fake":
has_besides_gguf = True
if has_gguf and has_besides_gguf:
- raise ValueError("gguf format is not compatible with other formats, please choose only one of them")
+ raise ValueError("Gguf format is not compatible with other formats, please choose only one of them")
if has_gguf and self.iters != 0 and self.bits != 3:
logger.warning(
"`iters=0` is recommended when exporting to GGUF format except for bits 3,"
@@ -539,7 +552,7 @@ class AutoRound(object):
" We are likely to release new algorithm for certain configurations in the future."
)
- ##check group_size 32 for auto_round
+ # Check group_size 32 for auto_round
if (
self.data_type == "int"
and hasattr(self, "formats")
@@ -561,7 +574,7 @@ class AutoRound(object):
):
if self.model.config.max_position_embeddings < self.seqlen:
logger.warning(
- f"change sequence length to {self.model.config.max_position_embeddings} "
+ f"Change sequence length to {self.model.config.max_position_embeddings} "
"due to the limitation of max_position_embeddings"
)
self.seqlen = min(self.seqlen, self.model.config.max_position_embeddings)
@@ -569,16 +582,16 @@ class AutoRound(object):
if self.seqlen is not None and hasattr(self.tokenizer, "model_max_length"):
if self.tokenizer.model_max_length < self.seqlen:
logger.warning(
- f"change sequence length to {self.tokenizer.model_max_length} "
+ f"Change sequence length to {self.tokenizer.model_max_length} "
"due to the limitation of model_max_length. "
"You can also try to increase the model_max_length to avoid this issue."
)
self.seqlen = min(self.seqlen, self.tokenizer.model_max_length)
if self.group_size == 0 and "fp8" not in self.data_type:
- logger.warning("group_size of 0 is not supported for data_type other than fp8 ")
+ logger.warning("`group_size==0` is not supported for data_type other than fp8 ")
- def parse_format_to_list(self, format: str) -> list:
+ def _parse_format_to_list(self, format: str) -> list:
"""Parses the format string into a list of formats.
This method checks the requested format(s) against the model's
@@ -670,7 +683,7 @@ class AutoRound(object):
if format == "fake":
return format
format = format.replace("q*_", f"q{self.bits}_")
- # only support to export afp8/nv_fp
+ # Only support to export afp8/nv_fp
if self.act_bits <= 8:
if not is_standard_fp(self.act_data_type) or self.act_dynamic:
if format == "llmcompressor":
@@ -743,7 +756,7 @@ class AutoRound(object):
self.orig_output_dir = output_dir
# check and update the format based on the current configuration
- format_list = self.parse_format_to_list(format)
+ format_list = self._parse_format_to_list(format)
self.formats = format_list
# If multiple formats are specified, enforce inplace=False
@@ -2695,7 +2708,7 @@ class AutoRound(object):
if not self.quantized:
logger.warning("please run autoround.quantize first")
return
- if format == "fake" or format == "qdq": ##TODO fix act quantizaiton later
+ if format == "fake" or format == "qdq": # TODO fix act quantization later
self.model = self.model.to("cpu")
self.model.save_pretrained(output_dir)
if self.tokenizer is not None:
@@ -2905,7 +2918,7 @@ class AutoRound(object):
return current_input_ids, current_input_others
-class AutoRoundOPT(AutoRound):
+class AutoRoundAdam(AutoRound):
"""Class for automatic rounding-based quantization with optimizers like adamw of a PyTorch model.
Args:
@@ -2999,7 +3012,7 @@ class AutoRoundOPT(AutoRound):
disable_opt_rtn: bool = False,
**kwargs,
):
- super(AutoRoundOPT, self).__init__(
+ super(AutoRoundAdam, self).__init__(
model=model,
tokenizer=tokenizer,
bits=bits,
@@ -3039,6 +3052,7 @@ class AutoRoundOPT(AutoRound):
device_map=device_map,
super_bits=super_bits,
super_group_size=super_group_size,
+ disable_opt_rtn=disable_opt_rtn,
**kwargs,
)
@@ -3082,142 +3096,3 @@ class AutoRoundOPT(AutoRound):
lr_schedule.step()
if is_optimum_habana_available():
htcore.mark_step()
-
-
-class AutoRoundAdam(AutoRoundOPT):
- """Class for automatic rounding-based quantization with optimizers like adamw of a PyTorch model.
- The default lr has been changed.
-
- Args:
- model: The PyTorch model to be quantized.
- tokenizer: An optional tokenizer for processing input data.
- bits (int): Number of bits for quantization (default is 4).
- group_size (int): Size of the quantization group (default is 128).
- sym (str): Whether symmetric quantization to be used (default is True).
- layer_config (dict): Configuration for weight quantization (default is None).
- batch_size (int): Batch size for training (default is 8).
- amp (bool): Whether to use automatic mixed precision (default is True).
- device: The device to be used for training (default is "auto").
- lr_scheduler: The learning rate scheduler to be used.
- dataset (Union[str, list, tuple, torch.utils.data.DataLoader]):
- The default dataset name (default is "NeelNanda/pile-10k").
- enable_quanted_input (bool): Whether to use quantized input data (default is True).
- enable_minmax_tuning (bool): Whether to enable min-max tuning (default is True).
- lr (float): The learning rate (default is 0.005).
- minmax_lr (float): The learning rate for min-max tuning (default is None).
- low_gpu_mem_usage (bool): Whether to use low GPU memory (default is False).
- low_cpu_mem_usage (bool): Whether to use low CPU memory (default is False).
- iters (int): Number of iterations (default is 200).
- seqlen (int): Length of the sequence.
- nsamples (int): Number of samples (default is 128).
- sampler (str): The sampling method (default is "rand").
- seed (int): The random seed (default is 42).
- nblocks (int): Number of blocks (default is 1).
- gradient_accumulate_steps (int): Number of gradient accumulation steps (default is 1).
- not_use_best_mse (bool): Whether to use mean squared error (default is False).
- dynamic_max_gap (int): The dynamic maximum gap (default is -1).
- data_type (str): The data type to be used (default is "int").
- optimizer: string or object
- scale_dtype (str): The data type of quantization scale to be used (default is "float16"), different kernels
- have different choices.
- act_bits (int): Number of bits for activation quantization. Default is 16.
- act_group_size (int): Group size for activation quantization. Default is None.
- act_sym (bool): Whether to use symmetric activation quantization. Default is None.
- act_data_type (str): Specifies the data type for activations.
- Defaults to None, in which case it inherits the weight data type.
- act_dynamic (bool): Whether to use dynamic activation quantization. Default is True.
- to_quant_block_names (str|list): A list whose elements are list of block's layer names to be quantized.
- enable_norm_bias_tuning (bool): Whether to enable fast norm/layer_bias tuning
- enable_torch_compile (bool): Whether to enable torch compile to optimize quant_block/layer function
- Returns:
- The quantized model.
- """
-
- def __init__(
- self,
- model: Union[torch.nn.Module, str],
- tokenizer=None,
- bits: int = 4,
- group_size: int = 128,
- sym: bool = True,
- layer_config=None,
- batch_size: int = 8,
- amp: bool = True,
- device: Union[str, torch.device, int] = 0,
- lr_scheduler=None,
- dataset: Union[str, list, tuple, torch.utils.data.DataLoader] = "NeelNanda/pile-10k",
- enable_quanted_input: bool = True,
- enable_minmax_tuning: bool = True,
- lr: float = None,
- minmax_lr: float = None,
- low_gpu_mem_usage: bool = False,
- low_cpu_mem_usage: int = 0,
- iters: int = 200,
- seqlen: int = 2048,
- nsamples: int = 128,
- sampler: str = "rand",
- seed: int = 42,
- nblocks: int = 1,
- gradient_accumulate_steps: int = 1,
- not_use_best_mse: bool = False,
- dynamic_max_gap: int = -1,
- data_type: str = "int",
- scale_dtype: str = "fp16",
- act_bits: int = 16,
- act_group_size: int = None,
- act_sym: bool = None,
- act_data_type: str = None,
- act_dynamic: bool = True,
- to_quant_block_names: Union[str, list] = None,
- enable_norm_bias_tuning: bool = False,
- enable_torch_compile: bool = False,
- device_map: Union[str, dict] = None,
- optimizer="AdamW",
- super_bits: int = None,
- super_group_size: int = None,
- disable_opt_rtn: bool = False,
- **kwargs,
- ):
- super(AutoRoundAdam, self).__init__(
- model=model,
- tokenizer=tokenizer,
- bits=bits,
- group_size=group_size,
- sym=sym,
- layer_config=layer_config,
- batch_size=batch_size,
- amp=amp,
- device=device,
- lr_scheduler=lr_scheduler,
- dataset=dataset,
- enable_quanted_input=enable_quanted_input,
- enable_minmax_tuning=enable_minmax_tuning,
- lr=lr,
- minmax_lr=minmax_lr,
- low_gpu_mem_usage=low_gpu_mem_usage,
- low_cpu_mem_usage=low_cpu_mem_usage,
- iters=iters,
- seqlen=seqlen,
- nsamples=nsamples,
- sampler=sampler,
- seed=seed,
- nblocks=nblocks,
- gradient_accumulate_steps=gradient_accumulate_steps,
- not_use_best_mse=not_use_best_mse,
- dynamic_max_gap=dynamic_max_gap,
- data_type=data_type,
- scale_dtype=scale_dtype,
- act_bits=act_bits,
- act_group_size=act_group_size,
- act_sym=act_sym,
- act_data_type=act_data_type,
- act_dynamic=act_dynamic,
- to_quant_block_names=to_quant_block_names,
- enable_norm_bias_tuning=enable_norm_bias_tuning,
- enable_torch_compile=enable_torch_compile,
- device_map=device_map,
- optimizer=optimizer,
- super_bits=super_bits,
- super_group_size=super_group_size,
- **kwargs,
- )
diff --git a/auto_round/inference/backend.py b/auto_round/inference/backend.py
index a4f578726..ea66a0389 100644
--- a/auto_round/inference/backend.py
+++ b/auto_round/inference/backend.py
@@ -115,7 +115,7 @@ BackendInfos["auto_gptq:exllamav2"] = BackendInfo(
group_size=[-1, 32, 64, 128, 256, 512, 1024, 2048],
feature_checks=[exllamav2_feature_check],
alias=["gptq", "auto_gptq", "exllamav2", "gptq:exllamav2", "auto_gptq:exllamav2"],
- requirements=["auto-gptq>=0.7.1"],
+ requirements=["torch<2.6.0", "auto-gptq>=0.7.1"],
)
BackendInfos["auto_gptq:tritonv2"] = BackendInfo(
@@ -128,7 +128,7 @@ BackendInfos["auto_gptq:tritonv2"] = BackendInfo(
priority=0,
feature_checks=[exllamav2_feature_check],
alias=["auto_gptq:tritonv2"],
- requirements=["auto-gptq>=0.7.1", "triton>=2.0"],
+ requirements=["torch<2.6.0", "auto-gptq>=0.7.1", "triton>=2.0"],
)
BackendInfos["auto_gptq:cuda"] = BackendInfo(
@@ -142,7 +142,10 @@ BackendInfos["auto_gptq:cuda"] = BackendInfo(
alias=["auto_gptq:cuda"],
dtype=["float16"],
convertable_format=["int32_zp"],
- requirements=["auto-gptq>=0.7.1"],
+ requirements=[
+ "torch<2.6.0",
+ "auto-gptq>=0.7.1",
+ ],
)
BackendInfos["auto_round:tritonv2"] = BackendInfo(
@@ -164,7 +167,7 @@ BackendInfos["auto_round:torch"] = BackendInfo(
dtype=["float16", "bfloat16"],
bits=[2, 3, 4, 8],
priority=0,
- feature_checks=[feature_multiply_checker_32],
+ feature_checks=[exllamav2_feature_check],
alias=["auto_round", "torch"],
requirements=["auto-round>=0.5.1"],
)
@@ -189,7 +192,7 @@ BackendInfos["auto_round:torch_zp"] = BackendInfo(
dtype=["float16", "bfloat16"],
bits=[2, 3, 4, 8],
priority=0,
- feature_checks=[feature_multiply_checker_32],
+ feature_checks=[exllamav2_feature_check],
alias=["torch", "torch_zp"],
requirements=["auto-round>=0.5.1"],
)
@@ -256,7 +259,7 @@ BackendInfos["qbits"] = BackendInfo(
alias=["itrex", "qbits"],
dtype=["float16", "bfloat16"],
convertable_format=["int32"],
- requirements=["intel-extension-for-transformers", "torch<2.7.0"],
+ requirements=["torch<2.7.0", "intel-extension-for-transformers"],
)
BackendInfos["qbits_zp"] = BackendInfo(
@@ -270,7 +273,7 @@ BackendInfos["qbits_zp"] = BackendInfo(
feature_checks=[],
alias=["itrex", "qbits"],
convertable_format=["int32_zp"],
- requirements=["intel-extension-for-transformers", "torch<2.7.0"],
+ requirements=["torch<2.7.0", "intel-extension-for-transformers"],
)
BackendInfos["auto_round:qbits_awq"] = BackendInfo(
@@ -283,7 +286,7 @@ BackendInfos["auto_round:qbits_awq"] = BackendInfo(
priority=1,
feature_checks=[],
alias=["itrex", "qbits"],
- requirements=["intel-extension-for-transformers", "torch<2.7.0"],
+ requirements=["torch<2.7.0", "intel-extension-for-transformers"],
)
BackendInfos["ipex_gptq"] = BackendInfo(
diff --git a/auto_round_extension/triton/qlinear_tritonv2.py b/auto_round_extension/triton/qlinear_tritonv2.py
index aabf90a1c..05b6fea15 100644
--- a/auto_round_extension/triton/qlinear_tritonv2.py
+++ b/auto_round_extension/triton/qlinear_tritonv2.py
@@ -179,8 +179,8 @@ class QuantLinear(nn.Module, TritonModuleMixin):
self.bits,
self.maxq,
)
- out = out.half().reshape(out_shape)
- out = out + self.bias if self.bias is not None else out
+ out = out.to(x.dtype).reshape(out_shape)
+ out = out + self.bias.to(x.dtype) if self.bias is not None else out
return out.to(x.dtype)
@classmethod
diff --git a/auto_round_extension/triton/qlinear_tritonv2_zp.py b/auto_round_extension/triton/qlinear_tritonv2_zp.py
index 222e1d013..36ecd6601 100644
--- a/auto_round_extension/triton/qlinear_tritonv2_zp.py
+++ b/auto_round_extension/triton/qlinear_tritonv2_zp.py
@@ -190,8 +190,8 @@ class QuantLinear(nn.Module, TritonModuleMixin):
self.bits,
self.maxq,
)
- out = out.half().reshape(out_shape)
- out = out + self.bias if self.bias is not None else out
+ out = out.to(x.dtype).reshape(out_shape)
+ out = out + self.bias.to(x.dtype) if self.bias is not None else out
return out.to(x.dtype)
@classmethod
diff --git a/test/test_cpu/test_autoopt.py b/test/test_cpu/test_autoopt.py
index fc6446dea..4f07c4e22 100644
--- a/test/test_cpu/test_autoopt.py
+++ b/test/test_cpu/test_autoopt.py
@@ -8,7 +8,7 @@ import torch
import transformers
from transformers import AutoModelForCausalLM, AutoTokenizer
-from auto_round import AutoRoundAdam, AutoRoundOPT
+from auto_round import AutoRoundAdam
class LLMDataLoader:
@@ -33,21 +33,6 @@ class TestAutoRound(unittest.TestCase):
shutil.rmtree("./saved", ignore_errors=True)
shutil.rmtree("runs", ignore_errors=True)
- def test_default(self):
- bits, group_size, sym = 4, 128, False
- autoround = AutoRoundOPT(
- self.model,
- self.tokenizer,
- bits=bits,
- group_size=group_size,
- sym=sym,
- iters=2,
- seqlen=10,
- dataset=self.llm_dataloader,
- to_quant_block_names=None,
- )
- autoround.quantize()
-
def test_Adam(self):
bits, group_size, sym = 4, 128, False
from auto_round.utils import get_block_names