Skip to content

PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度

[API CHANGE] Stage 1 add quant scheme and consolidate device and device_map (#774)

Wenhua Cheng8bcf7eab4bab
diff --git a/README.md b/README.md
index 819167905..434ad93bb 100644
--- a/README.md
+++ b/README.md
@@ -59,7 +59,7 @@ and [fbaldassarri](https://huggingface.co/fbaldassarri).
Delivers strong performance even at 2–3 bits [example models](https://huggingface.co/collections/OPEA/2-3-bits-67a5f0bc6b49d73c01b4753b), with leading results at 4 bits [benchmark](https://huggingface.co/spaces/Intel/low_bit_open_llm_leaderboard).
✅ **Ecosystem Integration**
-Seamlessly works with **Transformers, vLLM,** TorchAO, sglang(on going,[pr](https://github.com/sgl-project/sglang/pull/6226)) and more.
+Seamlessly works with **Transformers, vLLM,** and more.
✅ **Multiple Formats Export**
Support **AutoRound, AutoAWQ, AutoGPTQ, and GGUF** for maximum compatibility. Details are shown in [export formats](https://github.com/intel/auto-round/blob/main/docs/step_by_step.md#supported-export-formats)
diff --git a/auto_round/autoround.py b/auto_round/autoround.py
index f507d4b18..32213e570 100644
--- a/auto_round/autoround.py
+++ b/auto_round/autoround.py
@@ -18,10 +18,12 @@ import re
import sys
import time
import traceback
+from dataclasses import asdict
from typing import Any, Callable, Union
import accelerate
import torch
+from accelerate.big_modeling import dispatch_model, infer_auto_device_map
from torch import autocast
from tqdm import tqdm
from transformers import set_seed
@@ -30,6 +32,7 @@ from auto_round.data_type import QUANT_FUNC_WITH_DTYPE
from auto_round.data_type.utils import reshape_pad_tensor_by_group_size
from auto_round.export.export_to_gguf.config import GGUF_CONFIG, GGUF_INNER_CONFIG, ModelType
from auto_round.low_cpu_mem.utils import get_layers_before_block
+from auto_round.schemes import PRESET_SCHEMES, QuantizationScheme, preset_name_to_scheme
from auto_round.sign_sgd import SignSGD
from auto_round.special_model_handler import _handle_moe_model
from auto_round.utils import (
@@ -63,6 +66,7 @@ from auto_round.utils import (
get_layer_features,
get_layer_names_in_block,
get_lm_head_name,
+ get_max_vram,
get_module,
get_quant_keys,
get_shared_keys,
@@ -107,37 +111,34 @@ class AutoRound(object):
enable_torch_compile (bool): Whether to enable torch.compile for quant blocks/layers.
"""
+ bits: int | None
+ group_size: int | None
+ sym: bool | None
+ data_type: str | None
+ act_bits: int | None
+ act_group_size: int | None
+ act_sym: bool | None
+ act_data_type: str | None
+ act_dynamic: bool | None
+ super_bits: int | None
+ super_group_size: int | None
+
def __init__(
self,
model: Union[torch.nn.Module, str],
tokenizer=None,
- bits: int = 4,
- group_size: int = 128,
- sym: bool = True,
- layer_config: dict = None,
- batch_size: int = 8,
- amp: bool = True,
- device: Union[str, torch.device, int] = 0,
+ scheme: Union[str, dict, QuantizationScheme] = "W4A16",
+ layer_config: dict[str, Union[str, dict, QuantizationScheme]] = None,
dataset: Union[str, list, tuple, torch.utils.data.DataLoader] = "NeelNanda/pile-10k",
- enable_minmax_tuning: bool = True,
- lr: float = None,
- minmax_lr: float = None,
- low_gpu_mem_usage: bool = False,
iters: int = 200,
seqlen: int = 2048,
nsamples: int = 128,
- seed: int = 42,
+ batch_size: int = 8,
gradient_accumulate_steps: int = 1,
- data_type: str = "int",
- act_bits: int = 16,
- act_group_size: int = None,
- act_sym: bool = None,
- act_data_type: str = None,
- act_dynamic: bool = True,
+ low_gpu_mem_usage: bool = False,
+ device_map: Union[str, torch.device, int, dict] = 0,
enable_torch_compile: bool = False,
- device_map: Union[str, dict] = None,
- disable_opt_rtn: bool = False,
- enable_alg_ext: bool = False,
+ seed: int = 42,
**kwargs,
):
"""Initialize AutoRound with quantization and tuning configuration.
@@ -145,6 +146,7 @@ class AutoRound(object):
Args:
model (torch.nn.Module | str): Model object or model name to load.
tokenizer: Tokenizer for text processing. Required if `model` is not a string and `iters > 0`.
+ scheme (str| dict | QuantizationScheme ): A preset scheme that defines the quantization configurations
bits (int, optional): Weight quantization bits. Defaults to 4.
group_size (int, optional): Weight quantization group size. Defaults to 128.
sym (bool, optional): Symmetric weight quantization. Defaults to True.
@@ -173,7 +175,7 @@ class AutoRound(object):
disable_opt_rtn (bool, optional): Disable RTN-mode optimization (iters=0). Defaults to False.
enable_alg_ext (bool, optional): Enable algorithm extension (primarily for INT2). Defaults to False.
**kwargs: Backward compatible options:
- - lr_scheduler, sampler, not_use_best_mse, dynamic_max_gap,
+ - enable_alg_ext, lr, lr_scheduler, sampler, not_use_best_mse, dynamic_max_gap,
super_group_size, super_bits, scale_dtype ("fp16" etc.),
nblocks, low_cpu_mem_usage, to_quant_block_names,
enable_norm_bias_tuning, enable_quanted_input,
@@ -198,14 +200,21 @@ class AutoRound(object):
... # ...
... }
"""
+ self.scheme = None
+ self._parse_and_set_scheme(scheme, kwargs)
+
# Extra/legacy kwargs for backward compatibility
# Major version releases may pack them with extra configuration options
+ amp = kwargs.pop("amp", True)
+ lr = kwargs.pop("lr", None)
+ enable_alg_ext = kwargs.pop("enable_alg_ext", False)
+ enable_minmax_tuning = kwargs.pop("enable_minmax_tuning", True)
+ minmax_lr = kwargs.pop("minmax_lr", None)
+ disable_opt_rtn = kwargs.pop("disable_opt_rtn", False)
lr_scheduler = kwargs.pop("lr_scheduler", None)
sampler = kwargs.pop("sampler", "rand")
not_use_best_mse = kwargs.pop("not_use_best_mse", False)
dynamic_max_gap = kwargs.pop("dynamic_max_gap", -1)
- super_group_size = kwargs.pop("super_group_size", None)
- super_bits = kwargs.pop("super_bits", None)
scale_dtype = kwargs.pop("scale_dtype", "fp16")
nblocks = kwargs.pop("nblocks", 1)
low_cpu_mem_usage = kwargs.pop("low_cpu_mem_usage", False)
@@ -214,16 +223,46 @@ class AutoRound(object):
enable_quanted_input: bool = kwargs.pop("enable_quanted_input", True)
disable_deterministic_algorithms = kwargs.pop("disable_deterministic_algorithms", False)
static_kv_dtype = kwargs.pop("static_kv_dtype", None)
+ device = kwargs.pop("device", None)
+ if device is not None:
+ logger.warning("`device` is deprecated, please use `device_map` instead")
+
self.vlm = kwargs.pop("vlm") if "vlm" in kwargs else False
if kwargs:
logger.warning(f"unrecognized keys {list(kwargs.keys())} were passed. Please check them.")
- if device is not None and "," in str(device):
+ if device_map is not None and "," in str(device_map):
raise ValueError(
- "API does not support explicit set multiple devices,"
- " please set CUDA_VISIBLE_DEVICES yourself and use `device=auto` instead"
+ "API does not support explicit set multiple devices," " please set CUDA_VISIBLE_DEVICES=0,1 yourself"
)
+ if device_map is None:
+ device_map = 0
+
+ # Set device, must place after model loading
+ if isinstance(device_map, (str, torch.device, int)):
+ self.device = detect_device(device_map)
+ elif isinstance(device_map, dict) and device_map:
+ tmp_devices = []
+ for val in device_map.values():
+ if isinstance(val, (str, torch.device, int)): # could optimize
+ tmp_device = detect_device(self.device_map)
+ tmp_device = tmp_device.split(":")[0]
+ tmp_devices.append(tmp_device)
+ tmp_devices = list(set(tmp_devices))
+ if len(tmp_devices) > 1:
+ logger.warning(
+ f"there are multiple device types in the device_map, "
+ f"please make sure they are correct,use the first device {tmp_devices[0]} as the core device "
+ )
+
+ self.device = tmp_devices[0]
+
+ if isinstance(device_map, dict) and device_map:
+ self.device_map = device_map
+ else:
+ self.device_map = None
+ self._set_device_map_in_blocks(self.device_map)
if not disable_deterministic_algorithms:
if "CUBLAS_WORKSPACE_CONFIG" not in os.environ:
@@ -234,7 +273,9 @@ class AutoRound(object):
self.quantized = False
if isinstance(model, str):
model, tokenizer, low_cpu_mem_usage = llm_load_model(
- model, device=device, low_cpu_mem_mode=low_cpu_mem_usage
+ model,
+ device="cpu",
+ low_cpu_mem_mode=low_cpu_mem_usage, # always load cpu first
)
elif tokenizer is None and iters > 0:
raise ValueError("A tokenizer must be set for non-str model input")
@@ -250,49 +291,33 @@ class AutoRound(object):
self.tokenizer = tokenizer
self.shared_cache_keys = get_shared_keys(self.model)
- # Set quantization configs
- self.bits = bits
- self.data_type = data_type
- tmp_bits = infer_bits_by_data_type(self.data_type)
- if tmp_bits is not None and tmp_bits < 16 and tmp_bits != bits:
- logger.warning(f"'data_type' do not match the specified 'bits' setting. Resetting 'bits' to {tmp_bits}.")
- self.bits = tmp_bits
- if tmp_bits is not None and tmp_bits < 16:
- for supported_dtype in SUPPORTED_DTYPES: # to easily handle dtype mx_fp4 and layer_config={xxx:{bits:8}}
- if data_type.startswith(supported_dtype):
- if supported_dtype + str(tmp_bits) == self.data_type: # could not replace FP8_e4m3
- self.data_type = supported_dtype
- break
- self.group_size = group_size
- self.sym = sym
+ # Some other quantization configs
self.layer_config = {} if layer_config is None else layer_config
- self.to_quant_block_names = to_quant_block_names
+ # Check and convert to dict
+ for key, item in self.layer_config.items():
+ if isinstance(item, str):
+ item = asdict(preset_name_to_scheme(item.upper()))
+ self.layer_config[key] = item
+
+ if isinstance(item, QuantizationScheme):
+ config = asdict(item)
+ tmp_keys = copy.deepcopy(list(config.keys()))
+ for tmp_key in tmp_keys: ## Pop None value to be overridden
+ if config[tmp_key] is None:
+ config.pop(tmp_key)
+ self.layer_config[key] = config
+ elif isinstance(item, dict):
+ item_keys = item.keys()
+ expected_keys = list(QuantizationScheme.__annotations__.keys())
+ if item_keys not in expected_keys:
+ for item_key in item_keys:
+ if item_key not in expected_keys:
+ raise ValueError(
+ f"the key {item_key} in layer_config for layer {key} is invalid,"
+ f" only {expected_keys} are supported"
+ )
- # Activation
- self.act_group_size = act_group_size if act_group_size is not None else group_size
- self.act_bits = act_bits if act_bits is not None else self.bits
- self.act_sym = act_sym if act_sym is not None else self.sym
- self.act_dynamic = act_dynamic
- self.act_data_type = act_data_type
- if self.act_data_type is None:
- if self.data_type in SUPPORTED_DTYPES and self.act_bits < 16:
- self.act_data_type = data_type
- logger.info(f"activation adopts {data_type}")
- else:
- self.act_data_type = "float"
- tmp_act_bits = infer_bits_by_data_type(self.act_data_type)
- if tmp_act_bits is not None and tmp_act_bits < 16 and tmp_act_bits != self.act_bits:
- self.act_bits = tmp_act_bits
- logger.warning(
- f"`act_data_type` do not"
- f" match the specified 'act_bits' setting. Resetting 'act_bits' to {tmp_act_bits}."
- )
- if tmp_act_bits is not None and tmp_act_bits < 16:
- for supported_dtype in SUPPORTED_DTYPES: # to easily handle dtype mx_fp4 and layer_config={xxx:{bits:8}}
- if self.act_data_type.startswith(supported_dtype):
- if supported_dtype + str(tmp_act_bits) == self.act_data_type: # could not replace FP8_e4m3
- self.act_data_type = supported_dtype
- break
+ self.to_quant_block_names = to_quant_block_names
# Tuning hyperparameters
self.seed = seed
@@ -322,8 +347,6 @@ class AutoRound(object):
self.dynamic_max_gap = dynamic_max_gap
self.lr_scheduler = lr_scheduler
self.optimizer = self._get_optimizer(None)
- self.super_bits = super_bits
- self.super_group_size = super_group_size
self.disable_opt_rtn = disable_opt_rtn
self.is_packing_immediate = False # whether to pack the layer immediately after tuning
@@ -344,7 +367,7 @@ class AutoRound(object):
if unsupport_meta_device(model):
raise RuntimeError(
"AutoRound does not support parameters on meta device. "
- "Please use more GPUs by setting `--device 0,1,2,3` or just place the model on CPU."
+ "Please use more GPUs by setting `--device_map 0,1,2,3` or just place the model on CPU."
)
model = _handle_moe_model(model)
self.model = model.eval()
@@ -354,10 +377,6 @@ class AutoRound(object):
all_blocks = get_block_names(model)
self.quant_block_list = find_matching_blocks(model, all_blocks, self.to_quant_block_names)
- # Set device, must place after model loading
- self.device = detect_device(device) # must place after llm_load_model, because this one will convert auto
- self.device_map = device_map
- self._set_device_map_in_blocks(self.device_map)
self.scale_dtype = convert_dtype_str2torch(scale_dtype)
self._set_amp_dtype()
self.cache_device = torch.device("cpu") if self.low_gpu_mem_usage else self.device
@@ -416,6 +435,74 @@ class AutoRound(object):
"super_group_size",
]
+ def _parse_and_set_scheme(self, scheme: Union[str, dict, QuantizationScheme], kwargs) -> None:
+ """Parse and set the quantization scheme."""
+ if isinstance(scheme, QuantizationScheme):
+ scheme = asdict(scheme)
+ elif isinstance(scheme, dict):
+ scheme = scheme
+ elif isinstance(scheme, str):
+ scheme = scheme.upper()
+ self.scheme = scheme
+ scheme = asdict(preset_name_to_scheme(scheme))
+
+ scheme_keys = (
+ "bits",
+ "group_size",
+ "sym",
+ "data_type",
+ "act_bits",
+ "act_group_size",
+ "act_sym",
+ "act_data_type",
+ "act_dynamic",
+ "super_bits",
+ "super_group_size",
+ )
+ for key in scheme_keys:
+ if key in kwargs and kwargs[key] is not None:
+ setattr(self, key, kwargs[key])
+ else:
+ setattr(self, key, scheme.get(key, None))
+ kwargs.pop(key, None)
+ if self.act_dynamic is None:
+ self.act_dynamic = True
+
+ tmp_bits = infer_bits_by_data_type(self.data_type)
+ if tmp_bits is not None and tmp_bits < 16 and tmp_bits != self.bits:
+ logger.warning(f"'data_type' do not match the specified 'bits' setting. Resetting 'bits' to {tmp_bits}.")
+ self.bits = tmp_bits
+ if tmp_bits is not None and tmp_bits < 16:
+ for supported_dtype in SUPPORTED_DTYPES: # to easily handle dtype mx_fp4 and layer_config={xxx:{bits:8}}
+ if self.data_type.startswith(supported_dtype):
+ if supported_dtype + str(tmp_bits) == self.data_type: # could not replace FP8_e4m3
+ self.data_type = supported_dtype
+ break
+
+ self.act_group_size = self.act_group_size if self.act_group_size is not None else self.group_size
+ self.act_bits = self.act_bits if self.act_bits is not None else 16
+ self.act_sym = self.act_sym if self.act_sym is not None else self.sym
+
+ if self.act_data_type is None:
+ if self.data_type in SUPPORTED_DTYPES and self.act_bits < 16:
+ self.act_data_type = self.data_type
+ logger.info(f"activation adopts {self.data_type}")
+ else:
+ self.act_data_type = "float"
+ tmp_act_bits = infer_bits_by_data_type(self.act_data_type)
+ if tmp_act_bits is not None and tmp_act_bits < 16 and tmp_act_bits != self.act_bits:
+ self.act_bits = tmp_act_bits
+ logger.warning(
+ f"`act_data_type` do not"
+ f" match the specified 'act_bits' setting. Resetting 'act_bits' to {tmp_act_bits}."
+ )
+ if tmp_act_bits is not None and tmp_act_bits < 16:
+ for supported_dtype in SUPPORTED_DTYPES: # to easily handle dtype mx_fp4 and layer_config={xxx:{bits:8}}
+ if self.act_data_type.startswith(supported_dtype):
+ if supported_dtype + str(tmp_act_bits) == self.act_data_type: # could not replace FP8_e4m3
+ self.act_data_type = supported_dtype
+ break
+
def _adjust_torch_compile(self, enable_torch_compile: bool) -> None:
"""Sets the torch compile configuration for the tuning."""
self.enable_torch_compile = enable_torch_compile
@@ -976,8 +1063,6 @@ class AutoRound(object):
# Dispatch multi-GPU model if necessary
if hasattr(model, "hf_device_map") and len(model.hf_device_map) > 1:
- from accelerate.big_modeling import dispatch_model
-
dispatch_model(model, model.hf_device_map)
def register_act_hook(model):
@@ -1007,8 +1092,6 @@ class AutoRound(object):
try:
# Move model to target device
if hasattr(self.model, "hf_device_map") and len(self.model.hf_device_map) > 1:
- from accelerate.big_modeling import dispatch_model
-
dispatch_model(self.model, self.model.hf_device_map)
else:
model = model.to(self.device)
@@ -1099,20 +1182,26 @@ class AutoRound(object):
Raises:
NotImplementedError: If multiple non-fake GGUF formats are specified.
"""
- if not hasattr(self, "formats"):
+ gguf_scheme = False
+ if isinstance(self.scheme, str) and "gguf" in self.scheme.lower():
+ gguf_scheme = True
+
+ if not hasattr(self, "formats") and not gguf_scheme:
return False
- has_gguf: bool = any("gguf" in fmt for fmt in self.formats)
+ has_gguf: bool = gguf_scheme or any("gguf" in fmt for fmt in self.formats)
if not has_gguf:
return False
+ if hasattr(self, "formats"):
+ formats: list[str] = [fmt for fmt in self.formats if "fake" not in fmt]
+ if not (len(formats) == 1 and "gguf" in formats[0]):
+ raise NotImplementedError("Only one GGUF format can be set at a time.")
+ target_format: str = formats[0]
- formats: list[str] = [fmt for fmt in self.formats if "fake" not in fmt]
- if not (len(formats) == 1 and "gguf" in formats[0]):
- raise NotImplementedError("Only one GGUF format can be set at a time.")
+ else:
+ target_format = self.scheme.lower()
- target_format: str = formats[0]
tie_word_embeddings: bool = getattr(getattr(self.model, "config", None), "tie_word_embeddings", True)
-
for name, module in self.model.named_modules():
if isinstance(module, torch.nn.Embedding):
key: str = "lm_head" if tie_word_embeddings else "embedding"
@@ -1301,7 +1390,7 @@ class AutoRound(object):
if has_gguf_k and not self.disable_opt_rtn:
self._quant_rtn_with_imatrix(all_to_quantized_module_names)
elif self.act_bits <= 8 and check_need_act_calibration(
- self.act_dynamic, self.act_data_type
+ self.act_dynamic, self.act_data_type, self.act_bits
): # TODO, mixed datatype has bug
hook_handles = self._register_act_max_hook(self.model)
try:
@@ -1407,7 +1496,6 @@ class AutoRound(object):
convert_fp8_model_to_16b_model(block, dtype=self.amp_dtype)
# Dispatch model if needed
if self.device_map is not None:
- from accelerate import dispatch_model
from accelerate.hooks import AlignDevicesHook, add_hook_to_module
for _, m in block.named_modules():
@@ -1669,8 +1757,6 @@ class AutoRound(object):
enable_quanted_input = False
if hasattr(self.model, "hf_device_map") and len(self.model.hf_device_map) > 1 and enable_quanted_input:
- from accelerate.big_modeling import dispatch_model
-
dispatch_model(self.model, self.model.hf_device_map)
if enable_quanted_input:
@@ -2005,31 +2091,46 @@ class AutoRound(object):
layer_names = []
if self.low_gpu_mem_usage or (
- str(self.model.device == "cpu")
- and len(block_names) == 1
+ len(block_names) == 1
and len(layer_names) == 0
and not self.has_qlayer_outside_block
and (last_cache_name is None or last_cache_name in block_names)
):
- ## low_gpu_mem_usage or calibrate only the embedding layer, which is also very fast on CPU
+ # low_gpu_mem_usage or calibrate only the embedding layer, which is also very fast on CPU
all_inputs = self.cache_inter_data(block_names, nsamples, layer_names=[], last_cache_name=last_cache_name)
else:
try:
if not self.model.device.type == "meta":
if hasattr(self.model, "hf_device_map") and len(self.model.hf_device_map) > 1:
- pass
+ self.model = dispatch_model(self.model, device_map=self.model.hf_device_map)
else:
- self.model = self.model.to(self.device)
+ # Change this if new device is supported
+ if str(self.model.device) == "cpu" and (
+ self.device.startswith("xpu") or self.device.startswith("cuda")
+ ):
+ max_memory = get_max_vram() # TODO model is not evenly split
+ no_split_modules = getattr(self.model, "_no_split_modules", [])
+ device_map = infer_auto_device_map(
+ self.model, max_memory=max_memory, no_split_module_classes=no_split_modules
+ )
+
+ self.model = dispatch_model(self.model, device_map=device_map)
+ else:
+ self.model = self.model.to(self.device)
+
all_inputs = self.cache_inter_data(
block_names, nsamples, layer_names=layer_names, last_cache_name=last_cache_name
)
+ if hasattr(self.model, "hf_device_map") and len(self.model.hf_device_map) > 1:
+ accelerate.hooks.remove_hook_from_submodules(self.model)
+
except RuntimeError as e:
cuda_error_msg = traceback.format_exc()
try:
logger.info("switch to cpu to cache block inputs")
if self.has_qlayer_outside_block or self.__class__.__name__ == "AutoRoundMLLM":
logger.warning(
- "we strongly recommend using more GPUs in calibration."
+ "we recommend using more GPUs in calibration."
" Otherwise, some layers may fall back to `rtn` mode, which can affect accuracy."
)
if hasattr(self.model, "hf_device_map") and len(self.model.hf_device_map) > 1:
@@ -2308,10 +2409,8 @@ class AutoRound(object):
nsamples = len(inputs)
last_best_iter = 0
best_loss = torch.finfo(torch.float).max
- mse_loss = torch.nn.MSELoss().to(device)
scaler = self._get_scaler() # pylint: disable=assignment-from-none
init_loss = None
- # best_v, best_min_scale, best_max_scale = torch.tensor(0), torch.tensor(1.0), torch.tensor(1.0)
gradient_accumulate_steps = self.batch_size ##Force to low gpu
batch_size = 1 ##Force to low gpu
pick_samples = batch_size * gradient_accumulate_steps
@@ -2412,7 +2511,7 @@ class AutoRound(object):
for n, m in model.named_modules():
if (
hasattr(m, "act_dynamic")
- and check_need_act_calibration(m.act_dynamic, m.act_data_type)
+ and check_need_act_calibration(m.act_dynamic, m.act_data_type, m.act_bits)
and check_to_quantized(m)
):
hook = m.register_forward_hook(get_act_max_hook)
@@ -2424,9 +2523,10 @@ class AutoRound(object):
config = self.layer_config[n]
act_dynamic = config.get("act_dynamic", True)
act_data_type = config.get("act_data_type", None)
+ act_bits = config.get("act_data_type", 16)
if (
config["bits"] <= 8
- and check_need_act_calibration(act_dynamic, act_data_type)
+ and check_need_act_calibration(act_dynamic, act_data_type, act_bits)
and check_to_quantized(config)
):
hook = m.register_forward_hook(get_act_max_hook)
@@ -2461,8 +2561,6 @@ class AutoRound(object):
set_module(block, n, new_layer)
if self.device_map is not None:
- from accelerate import dispatch_model
-
for n, m in block.named_modules():
if len(list(m.children())) != 0 or not hasattr(m, "tuning_device"):
continue
@@ -3054,6 +3152,7 @@ class AutoRoundAdam(AutoRound):
Args:
model: The PyTorch model to be quantized.
tokenizer: An optional tokenizer for processing input data.
+ scheme (str| dict | QuantizationScheme ): A preset scheme that defines the quantization configurations
bits (int): Number of bits for quantization (default is 4).
group_size (int): Size of the quantization group (default is 128).
sym (bool): Whether sym to be used (default is True).
@@ -3097,92 +3196,52 @@ class AutoRoundAdam(AutoRound):
The quantized model.
"""
+ bits: int | None
+ group_size: int | None
+ sym: bool | None
+ data_type: str | None
+ act_bits: int | None
+ act_group_size: int | None
+ act_sym: bool | None
+ act_data_type: str | None
+ act_dynamic: bool | None
+ super_bits: int | None
+ super_group_size: int | None
+
def __init__(
self,
model: Union[torch.nn.Module, str],
tokenizer=None,
- bits: int = 4,
- group_size: int = 128,
- sym: bool = True,
- layer_config=None,
- batch_size: int = 8,
- amp: bool = True,
- device: Union[str, torch.device, int] = 0,
- lr_scheduler=None,
+ scheme: Union[str, dict, QuantizationScheme] = "W4A16",
+ layer_config: dict[str, Union[str, dict, QuantizationScheme]] = None,
dataset: Union[str, list, tuple, torch.utils.data.DataLoader] = "NeelNanda/pile-10k",
- enable_quanted_input: bool = True,
- enable_minmax_tuning: bool = True,
- lr: float = None,
- minmax_lr: float = None,
- low_gpu_mem_usage: bool = False,
- low_cpu_mem_usage: int = 0,
iters: int = 200,
seqlen: int = 2048,
nsamples: int = 128,
- sampler: str = "rand",
- seed: int = 42,
- nblocks: int = 1,
+ batch_size: int = 8,
gradient_accumulate_steps: int = 1,
- not_use_best_mse: bool = False,
- dynamic_max_gap: int = -1,
- data_type: str = "int",
- scale_dtype: str = "fp16",
- act_bits: int = 16,
- act_group_size: int = None,
- act_sym: bool = None,
- act_data_type: str = None,
- act_dynamic: bool = True,
- to_quant_block_names: Union[str, list] = None,
- enable_norm_bias_tuning: bool = False,
+ low_gpu_mem_usage: bool = False,
+ device_map: Union[str, int, torch.device, dict] = 0,
enable_torch_compile: bool = False,
- device_map: Union[str, dict] = None,
+ seed: int = 42,
optimizer="AdamW",
- super_bits: int = None,
- super_group_size: int = None,
- disable_opt_rtn: bool = False,
**kwargs,
):
super(AutoRoundAdam, self).__init__(
model=model,
tokenizer=tokenizer,
- bits=bits,
- group_size=group_size,
- sym=sym,
+ scheme=scheme,
layer_config=layer_config,
batch_size=batch_size,
- amp=amp,
- device=device,
- lr_scheduler=lr_scheduler,
dataset=dataset,
- enable_quanted_input=enable_quanted_input,
- enable_minmax_tuning=enable_minmax_tuning,
- lr=lr,
- minmax_lr=minmax_lr,
low_gpu_mem_usage=low_gpu_mem_usage,
- low_cpu_mem_usage=low_cpu_mem_usage,
iters=iters,
seqlen=seqlen,
nsamples=nsamples,
- sampler=sampler,
seed=seed,
- nblocks=nblocks,
gradient_accumulate_steps=gradient_accumulate_steps,
- not_use_best_mse=not_use_best_mse,
- dynamic_max_gap=dynamic_max_gap,
- data_type=data_type,
- scale_dtype=scale_dtype,
- act_bits=act_bits,
- act_group_size=act_group_size,
- act_sym=act_sym,
- act_data_type=act_data_type,
- act_dynamic=act_dynamic,
- to_quant_block_names=to_quant_block_names,
- enable_norm_bias_tuning=enable_norm_bias_tuning,
enable_torch_compile=enable_torch_compile,
device_map=device_map,
- super_bits=super_bits,
- super_group_size=super_group_size,
- disable_opt_rtn=disable_opt_rtn,
**kwargs,
)
diff --git a/auto_round/mllm/eval.py b/auto_round/mllm/eval.py
index 4d3f5e8e8..e08c88ebd 100644
--- a/auto_round/mllm/eval.py
+++ b/auto_round/mllm/eval.py
@@ -52,9 +52,8 @@ import time
from functools import partial
import numpy as np
-import pandas as pd
-from ..utils import LazyImport, logger
+from auto_round.utils import LazyImport, logger
vlmeval = LazyImport("vlmeval")
@@ -304,6 +303,8 @@ def mllm_eval(
if mode == "all":
eval_results = dataset.evaluate(result_file, **judge_kwargs)
+ import pandas as pd
+
if eval_results is not None:
if not isinstance(eval_results, dict) and not isinstance(eval_results, pd.DataFrame):
raise TypeError("Unsupported eval result type")
diff --git a/auto_round/schemes.py b/auto_round/schemes.py
new file mode 100644
index 000000000..e556acd29
--- /dev/null
+++ b/auto_round/schemes.py
@@ -0,0 +1,175 @@
+# Copyright (c) 2025 Intel Corporation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+import copy
+from copy import deepcopy
+from dataclasses import dataclass
+from typing import Optional
+
+__all__ = ["QuantizationScheme", "preset_name_to_scheme"]
+
+
+@dataclass
+class QuantizationScheme:
+ bits: int = 4
+ group_size: int = 128
+ sym: bool = True
+ data_type: str = "int"
+ act_bits: Optional[int] = None
+ act_group_size: Optional[int] = None
+ act_sym: Optional[bool] = None
+ act_data_type: Optional[str] = None
+ act_dynamic: Optional[bool] = None
+ super_bits: Optional[int] = None
+ super_group_size: Optional[int] = None
+
+ @classmethod
+ def from_dict(cls, config: dict):
+ return cls(**config)
+
+
+def preset_name_to_scheme(name: str) -> QuantizationScheme:
+ """Get a QuantizationScheme instance from a preset scheme name."""
+ name = name.upper()
+
+ if name not in PRESET_SCHEMES:
+ raise KeyError(f"Unknown preset scheme name {name}, " f"available names: {list(PRESET_SCHEMES.keys())}")
+
+ scheme_args = deepcopy(PRESET_SCHEMES[name])
+ return scheme_args
+
+
+def is_preset_scheme(name: str) -> bool:
+ """Check if the given name is a preset scheme name."""
+ return name.upper() in PRESET_SCHEMES
+
+
+W4A16 = QuantizationScheme.from_dict(
+ {
+ "bits": 4,
+ "sym": True,
+ "group_size": 128,
+ "data_type": "int",
+ "act_bits": 16,
+ }
+)
+
+W2A16 = QuantizationScheme.from_dict(
+ {
+ "bits": 2,
+ "sym": True,
+ "group_size": 128,
+ "data_type": "int",
+ "act_bits": 16,
+ }
+)
+
+W3A16 = QuantizationScheme.from_dict(
+ {
+ "bits": 3,
+ "sym": True,
+ "group_size": 128,
+ "data_type": "int",
+ "act_bits": 16,
+ }
+)
+
+W8A16 = QuantizationScheme.from_dict(
+ {
+ "bits": 8,
+ "sym": True,
+ "group_size": 128,
+ "data_type": "int",
+ "act_bits": 16,
+ }
+)
+
+MXFP4 = QuantizationScheme.from_dict(
+ {
+ "bits": 4,
+ "group_size": 32,
+ "data_type": "mx_fp",
+ "act_bits": 4,
+ "act_data_type": "mx_fp_rceil",
+ }
+)
+
+MXFP8 = QuantizationScheme.from_dict(
+ {
+ "bits": 8,
+ "group_size": 32,
+ "data_type": "mx_fp",
+ "act_bits": 8,
+ "act_data_type": "mx_fp_rceil",
+ }
+)
+
+NVFP4 = QuantizationScheme.from_dict(
+ {
+ "bits": 4,
+ "group_size": 16,
+ "data_type": "nv_fp",
+ "act_bits": 4,
+ "act_data_type": "nv_fp_with_static_gs",
+ }
+)
+
+FPW8A16 = QuantizationScheme.from_dict(
+ {
+ "bits": 8,
+ "group_size": 0,
+ "data_type": "fp",
+ "act_bits": 16,
+ "act_data_type": "fp",
+ }
+)
+
+# FP8 = asdict(QuantArgs.from_dict({
+# "bits": 8,
+# "group_size": 128,
+# "data_type": "fp",
+# "act_bits": 8,
+# "act_data_type": "fp",
+# }))
+
+FPW8_STATIC = QuantizationScheme.from_dict(
+ {
+ "bits": 8,
+ "group_size": -1,
+ "data_type": "fp",
+ "act_bits": 8,
+ "act_group_size": 0,
+ "act_data_type": "fp",
+ "act_dynamic": False,
+ }
+)
+
+PRESET_SCHEMES = {
+ "W4A16": W4A16,
+ "W2A16": W2A16,
+ "W3A16": W3A16,
+ "W8A16": W8A16,
+ "MXFP4": MXFP4,
+ "MXFP8": MXFP8,
+ "NVFP4": NVFP4,
+ "FPW8A16": FPW8A16,
+ "FPW8_STATIC": FPW8_STATIC,
+}
+from auto_round.export.export_to_gguf.config import GGUF_CONFIG
+
+for key, val in GGUF_CONFIG.items():
+ value = copy.deepcopy(val)
+ value.pop("mostly", None)
+ value.pop("embedding", None)
+ value.pop("lm_head", None)
+ PRESET_SCHEMES[key.upper()] = QuantizationScheme.from_dict(value)
diff --git a/auto_round/script/llm.py b/auto_round/script/llm.py
index 3caefcb85..0aaadef83 100644
--- a/auto_round/script/llm.py
+++ b/auto_round/script/llm.py
@@ -32,6 +32,7 @@ import os
import re
import sys
+from auto_round.schemes import PRESET_SCHEMES
from auto_round.utils import (
clear_memory,
get_device_and_parallelism,
@@ -39,7 +40,6 @@ from auto_round.utils import (
get_model_dtype,
infer_bits_by_data_type,
set_cuda_visible_devices,
- str2bool,
)
@@ -55,11 +55,35 @@ class BasicArgumentParser(argparse.ArgumentParser):
self.add_argument("--eval", action="store_true", help="whether to use eval only mode")
- self.add_argument("--bits", default=4, type=int, help="number of weight bits")
+ self.add_argument(
+ "--scheme",
+ default="W4A16",
+ type=str,
+ # choices=["W4A16", "W2A16", "W3A16", "W8A16", "MXFP4", "MXFP8", "NVFP4", "FPW8A16", "FPW8_STATIC"],
+ help="quantization scheme",
+ )
+
+ self.add_argument("--bits", default=None, type=int, help="number of weight bits")
+ self.add_argument("--group_size", default=None, type=int, help="group size")
+ self.add_argument("--asym", action="store_true", help="whether to use asym quantization")
+ self.add_argument("--data_type", "--dtype", default=None, help="data type for tuning, 'int', 'mx_fp' and etc")
+ self.add_argument("--act_bits", default=None, type=int, help="activation bits")
+ self.add_argument("--act_group_size", default=None, type=int, help="activation group size")
+ self.add_argument(
+ "--super_group_size", default=None, type=int, help="the number of super group size when use double quant."
+ )
+
+ self.add_argument(
+ "--super_bits", default=None, type=int, help="number of scale and mins quant bits for double quant."
+ )
+ self.add_argument("--act_data_type", "--act_dtype", default=None, type=str, help="activation data type")
+
+ self.add_argument("--disable_act_dynamic", action="store_true", help="activation static quantization")
self.add_argument("--eval_bs", default=None, type=int, help="batch size in evaluation")
self.add_argument(
+ "--device_map",
"--device",
"--devices",
default="0",
@@ -71,8 +95,6 @@ class BasicArgumentParser(argparse.ArgumentParser):
"set --device 0,1,2 to use multiple cards.",
)
- self.add_argument("--asym", action="store_true", help="whether to use asym quantization")
-
self.add_argument(
"--dataset", default="NeelNanda/pile-10k", type=str, help="the dataset for quantization training"
)
@@ -96,8 +118,6 @@ class BasicArgumentParser(argparse.ArgumentParser):
self.add_argument("--format", default="auto_round", type=str, help="the format to save the model")
- self.add_argument("--data_type", "--dtype", default="int", help="data type for tuning, 'int', 'mx_fp' and etc")
-
self.add_argument(
"--scale_dtype",
default="fp16",
@@ -175,10 +195,6 @@ class BasicArgumentParser(argparse.ArgumentParser):
help="force to convert the dtype, some backends supports fp16 dtype better",
)
- self.add_argument("--act_bits", default=16, type=int, help="activation bits")
-
- self.add_argument("--act_group_size", default=None, type=int, help="activation group size")
-
self.add_argument(
"--fp_layers", default="", type=str, help="list of Layer names to maintain original data type"
)
@@ -200,24 +216,10 @@ class BasicArgumentParser(argparse.ArgumentParser):
self.add_argument("--enable_alg_ext", action="store_true", help="whether to enable probably better algorithm")
- self.add_argument("--act_data_type", "--act_dtype", default=None, type=str, help="activation data type")
-
- self.add_argument("--disable_act_dynamic", action="store_true", help="activation static quantization")
-
self.add_argument(
"--disable_deterministic_algorithms", action="store_true", help="disable torch deterministic algorithms."
)
- self.add_argument("--device_map", default=None, type=str, help="device_map for block in tuning phase")
-
- self.add_argument(
- "--super_group_size", default=None, type=int, help="the number of super group size when use double quant."
- )
-
- self.add_argument(
- "--super_bits", default=None, type=int, help="number of scale and mins quant bits for double quant."
- )
-
self.add_argument(
"--eval_model_dtype", default=None, type=str, help="the torch_dytpe to load the model for evaluation."
)
@@ -263,6 +265,7 @@ class EvalArgumentParser(argparse.ArgumentParser):
)
self.add_argument("--mllm", action="store_true", help="whether to eval multi-modal model.")
self.add_argument(
+ "--device_map",
"--device",
"--devices",
default="0",
@@ -294,8 +297,6 @@ class EvalArgumentParser(argparse.ArgumentParser):
def setup_parser():
parser = BasicArgumentParser()
- parser.add_argument("--group_size", default=128, type=int, help="group size")
-
parser.add_argument("--batch_size", "--train_bs", "--bs", default=8, type=int, help="train batch size")
parser.add_argument("--iters", "--iter", default=200, type=int, help="iteration to tune each block")
@@ -317,8 +318,6 @@ def setup_parser():
def setup_best_parser():
parser = BasicArgumentParser()
- parser.add_argument("--group_size", default=128, type=int, help="group size")
-
parser.add_argument("--batch_size", "--train_bs", "--bs", default=8, type=int, help="train batch size")
parser.add_argument("--iters", "--iter", default=1000, type=int, help="iterations to tune each block")
@@ -342,8 +341,6 @@ def setup_best_parser():
def setup_light_parser():
parser = BasicArgumentParser()
- parser.add_argument("--group_size", default=128, type=int, help="group size")
-
parser.add_argument("--batch_size", "--train_bs", "--bs", default=8, type=int, help="train batch size")
parser.add_argument("--iters", "--iter", default=50, type=int, help="iterations to tune each block")
@@ -366,8 +363,6 @@ def setup_light_parser():
def setup_fast_parser():
parser = BasicArgumentParser()
- parser.add_argument("--group_size", default=128, type=int, help="group size")
-
parser.add_argument("--batch_size", "--train_bs", "--bs", default=4, type=int, help="train batch size")
parser.add_argument("--iters", default=200, type=int, help="iterations to tune each block")
@@ -399,9 +394,7 @@ def tune(args):
if args.eval_bs is None:
args.eval_bs = "auto"
-
- import transformers
- from transformers import AutoConfig, AutoModel, AutoModelForCausalLM, AutoTokenizer
+ from transformers import AutoConfig, AutoModelForCausalLM, AutoTokenizer
from auto_round.utils import detect_device, get_library_version, logger
@@ -424,9 +417,9 @@ def tune(args):
if "marlin" in args.format and args.asym is True:
raise RuntimeError("marlin backend only supports sym quantization, please remove --asym")
- ##must set this before import torch
- set_cuda_visible_devices(args.device)
- device_str, use_auto_mapping = get_device_and_parallelism(args.device)
+ # Must set this before import torch
+ set_cuda_visible_devices(args.device_map)
+ device_str, use_auto_mapping = get_device_and_parallelism(args.device_map)
import torch
@@ -446,8 +439,8 @@ def tune(args):
if args.mllm:
model, processor, tokenizer, image_processor = mllm_load_model(
model_name,
- device=device_str,
- use_auto_mapping=use_auto_mapping,
+ device="cpu",
+ use_auto_mapping=False,
trust_remote_code=not args.disable_trust_remote_code,
model_dtype=args.model_dtype,
)
@@ -455,9 +448,9 @@ def tune(args):
else:
model, tokenizer, low_cpu_mem_usage = llm_load_model(
model_name,
- use_auto_mapping=use_auto_mapping,
+ use_auto_mapping=False,
trust_remote_code=not args.disable_trust_remote_code,
- device=device_str,
+ device="cpu",
low_cpu_mem_mode=args.low_cpu_mem_mode,
low_cpu_mem_tmp_dir=args.low_cpu_mem_tmp_dir,
model_dtype=args.model_dtype,
@@ -495,7 +488,7 @@ def tune(args):
and "awq" not in format
and "llmcompressor" not in format
):
- ##TODO gptq could support some mixed precision config
+ # TODO gptq could support some mixed precision config
logger.warning(f"mixed precision exporting does not support {format} currently")
lm_head_layer_name = "lm_head"
@@ -506,7 +499,7 @@ def tune(args):
if config.tie_word_embeddings and hasattr(model, "_tied_weights_keys"):
tied_keys = model._tied_weights_keys
for item in tied_keys:
- if lm_head_layer_name in item: ##TODO extend to encoder-decoder layer, seq classification model
+ if lm_head_layer_name in item: # TODO extend to encoder-decoder layer, seq classification model
args.quant_lm_head = False
logger.warning(
"reset `quant_lm_head` to `False` as quantizing lm_head with tied weights has not been "
@@ -533,13 +526,22 @@ def tune(args):
logger.warning(f"The AutoAWQ format may not be supported due to {info}")
enable_torch_compile = True if "--enable_torch_compile" in sys.argv else False
-
+ sym = None # the default value should be None now
+ if args.asym: # if the scheme is asym, how to set it to sym is an issue
+ sym = False
+ act_dynamic = None
+ if args.disable_act_dynamic:
+ act_dynamic = False
+ scheme = args.scheme.upper()
+ if scheme not in PRESET_SCHEMES:
+ raise ValueError(f"{scheme} is not supported. only {PRESET_SCHEMES.keys()} are supported ")
autoround = round(
model=model,
tokenizer=tokenizer,
+ scheme=scheme,
bits=args.bits,
group_size=args.group_size,
- sym=not args.asym,
+ sym=sym,
batch_size=args.batch_size,
dataset=args.dataset,
seqlen=args.seqlen,
@@ -548,7 +550,6 @@ def tune(args):
lr=args.lr,
minmax_lr=args.minmax_lr,
enable_quanted_input=not args.disable_quanted_input,
- device=device_str,
amp=not args.disable_amp,
nsamples=args.nsamples,
seed=args.seed,
@@ -566,7 +567,7 @@ def tune(args):
to_quant_block_names=args.to_quant_block_names,
enable_torch_compile=enable_torch_compile,
act_data_type=args.act_data_type,
- act_dynamic=not args.disable_act_dynamic,
+ act_dynamic=act_dynamic,
device_map=args.device_map,
super_group_size=args.super_group_size,
super_bits=args.super_bits,
@@ -723,7 +724,7 @@ def tune(args):
from auto_round.eval.evaluation import simple_evaluate
tasks, model_args, device_str = _eval_init(
- args.tasks, eval_folder, args.device, args.disable_trust_remote_code, dtype=eval_model_dtype
+ args.tasks, eval_folder, args.device_map, args.disable_trust_remote_code, dtype=eval_model_dtype
)
st = time.time()
if "llama" in args.model.lower():
@@ -753,7 +754,7 @@ def eval(args):
import time
tasks, model_args, device_str = _eval_init(
- args.tasks, args.model, args.device, args.disable_trust_remote_code, args.eval_model_dtype
+ args.tasks, args.model, args.device_map, args.disable_trust_remote_code, args.eval_model_dtype
)
# load after _eval_int in order to make sure import torch after set CUDA_VISIBLE_DEVICES
diff --git a/auto_round/script/mllm.py b/auto_round/script/mllm.py
index 1f55402df..6ab5a264f 100644
--- a/auto_round/script/mllm.py
+++ b/auto_round/script/mllm.py
@@ -16,6 +16,7 @@ import argparse
import os
import sys
+from auto_round.schemes import PRESET_SCHEMES
from auto_round.utils import (
clear_memory,
get_device_and_parallelism,
@@ -40,7 +41,18 @@ class BasicArgumentParser(argparse.ArgumentParser):
self.add_argument("--eval", action="store_true", help="whether to use eval only mode.")
- self.add_argument("--bits", default=4, type=int, help="weight bits")
+ self.add_argument(
+ "--scheme",
+ default="W4A16",
+ type=str,
+ help="quantization scheme",
+ )
+
+ self.add_argument("--bits", default=None, type=int, help="number of weight bits")
+ self.add_argument("--group_size", default=None, type=int, help="group size")
+ self.add_argument("--asym", action="store_true", help="whether to use asym quantization")
+ self.add_argument("--data_type", "--dtype", default=None, help="data type for tuning, 'int', 'mx_fp' and etc")
+ self.add_argument("--act_bits", default=None, type=int, help="activation bits")
self.add_argument("--eval_bs", default=None, type=int, help="batch size in evaluation")
@@ -56,8 +68,6 @@ class BasicArgumentParser(argparse.ArgumentParser):
"set --device 0,1,2 to use multiple cards.",
)
- self.add_argument("--asym", action="store_true", help="whether to use asym quantization")
-
self.add_argument(
"--dataset",
type=str,
@@ -91,8 +101,6 @@ class BasicArgumentParser(argparse.ArgumentParser):
self.add_argument("--format", default="auto_round", type=str, help="the format to save the model")
- self.add_argument("--data_type", "--dtype", default="int", help="data type for tuning, 'int', 'mx_fp' and etc")
-
self.add_argument(
"--scale_dtype",
default="fp16",
@@ -154,8 +162,6 @@ class BasicArgumentParser(argparse.ArgumentParser):
help="force to convert the dtype, some backends supports fp16 dtype better",
)
- self.add_argument("--act_bits", default=16, type=int, help="activation bits")
-
self.add_argument("--fp_layers", default="", type=str, help="layers to maintain original data type")
self.add_argument(
@@ -221,8 +227,6 @@ class BasicArgumentParser(argparse.ArgumentParser):
def setup_parser():
parser = BasicArgumentParser()
- parser.add_argument("--group_size", default=128, type=int, help="group size")
-
parser.add_argument("--batch_size", "--train_bs", "--bs", default=8, type=int, help="train batch size")
parser.add_argument("--iters", "--iter", default=200, type=int, help=" iters")
@@ -323,9 +327,9 @@ def tune(args):
if format not in SUPPORTED_FORMATS:
raise ValueError(f"{format} is not supported, we only support {SUPPORTED_FORMATS}")
- ##must set this before import torch
- set_cuda_visible_devices(args.device)
- device_str, use_auto_mapping = get_device_and_parallelism(args.device)
+ # Must set this before import torch
+ set_cuda_visible_devices(args.device_map)
+ device_str, use_auto_mapping = get_device_and_parallelism(args.device_map)
import torch
@@ -342,8 +346,9 @@ def tune(args):
model, processor, tokenizer, image_processor = mllm_load_model(
model_name,
+ device="cpu",
torch_dtype=torch_dtype,
- use_auto_mapping=use_auto_mapping,
+ use_auto_mapping=False,
trust_remote_code=not args.disable_trust_remote_code,
model_dtype=args.model_dtype,
)
@@ -438,16 +443,26 @@ def tune(args):
"trust_remote_code": not args.disable_trust_remote_code,
"model_dtype": args.model_dtype,
}
+
+ sym = None # the default value should be None now
+ if args.asym: # if the scheme is asym, how to set it to sym is an issue
+ sym = False
+
+ scheme = args.scheme.upper()
+ if scheme not in PRESET_SCHEMES:
+ raise ValueError(f"{scheme} is not supported. only {PRESET_SCHEMES.keys()} are supported ")
+
autoround = round(
model,
tokenizer,
+ scheme=args.scheme,
processor=processor,
image_processor=image_processor,
dataset=args.dataset,
extra_data_dir=args.extra_data_dir,
bits=args.bits,
group_size=args.group_size,
- sym=not args.asym,
+ sym=sym,
batch_size=args.batch_size,
seqlen=seqlen,
nblocks=args.nblocks,
@@ -459,7 +474,6 @@ def tune(args):
truncation=args.truncation,
nsamples=args.nsamples,
low_gpu_mem_usage=args.low_gpu_mem_usage,
- device=device_str,
seed=args.seed,
gradient_accumulate_steps=args.gradient_accumulate_steps,
scale_dtype=args.scale_dtype,
@@ -503,8 +517,8 @@ def tune(args):
def vlmeval(args):
- set_cuda_visible_devices(args.device)
- device_str, parallelism = get_device_and_parallelism(args.device)
+ set_cuda_visible_devices(args.device_map)
+ device_str, parallelism = get_device_and_parallelism(args.device_map)
if parallelism:
os.environ["AUTO_SPLIT"] = "1"
if isinstance(args.tasks, str):
@@ -582,8 +596,8 @@ def setup_lmms_parser():
def lmms_eval(args):
- set_cuda_visible_devices(args.device)
- device_str, parallelism = get_device_and_parallelism(args.device)
+ set_cuda_visible_devices(args.device_map)
+ device_str, parallelism = get_device_and_parallelism(args.device_map)
from auto_round.mllm import lmms_eval
diff --git a/auto_round/utils.py b/auto_round/utils.py
index 2f93dddcc..9c94f9216 100644
--- a/auto_round/utils.py
+++ b/auto_round/utils.py
@@ -547,7 +547,7 @@ def detect_device_count():
return 0
-def detect_device(device=None):
+def detect_device(device: Union[str, int, torch.device] = None) -> str:
"""Detects the appropriate computation device.
This function determines the device to use for computations. It can take
@@ -2191,8 +2191,13 @@ def get_reciprocal(tensor):
return torch.where(tensor != 0, 1 / tensor, torch.zeros_like(tensor))
-def check_need_act_calibration(is_act_dynamic, act_data_type=None):
- if not is_act_dynamic:
+def check_need_act_calibration(
+ is_act_dynamic: Union[bool, None], act_data_type: Union[str, None] = None, act_bits: int = 16
+) -> bool:
+ if act_bits > 8:
+ return False
+ # None is dynamic
+ if is_act_dynamic is not None and not is_act_dynamic:
return True
if act_data_type is not None and "static" in act_data_type:
return True
@@ -2552,3 +2557,23 @@ def is_static_wfp8afp8(ar):
if is_wfp8afp8(ar):
return True
return False
+
+
+def get_max_vram(ratio: float = 0.9) -> dict:
+ max_memory = {}
+ if torch.cuda.is_available(): # NVIDIA CUDA
+ num_devices = torch.cuda.device_count()
+ for i in range(num_devices):
+ total_mem = torch.cuda.get_device_properties(i).total_memory
+ max_mem_gb = int(total_mem / 1024**3 * ratio)
+ max_memory[i] = f"{max_mem_gb}GiB"
+ elif torch.xpu.is_available(): # TODO need verification
+ num_devices = torch.xpu.device_count()
+ for i in range(num_devices):
+ total_mem = torch.xpu.get_device_properties(i).total_memory
+ max_mem_gb = int(total_mem / 1024**3 * ratio)
+ max_memory[i] = f"{max_mem_gb}GiB"
+
+ else:
+ raise RuntimeError("No CUDA or XPU devices found.")
+ return max_memory
diff --git a/test/test_cpu/test_autoround.py b/test/test_cpu/test_autoround.py
index 43dbe2365..ccdf5b419 100644
--- a/test/test_cpu/test_autoround.py
+++ b/test/test_cpu/test_autoround.py
@@ -636,6 +636,15 @@ class TestAutoRound(unittest.TestCase):
):
raise ValueError("mixed bits is not correct")
+ def test_invalid_layer_config(self):
+ with self.assertRaises(ValueError):
+ layer_config = {"model.decoder.layers.2.self_attnx": {"bits": 2}}
+ ar = AutoRound("facebook/opt-125m", scheme="W3A16", nsamples=1, iters=1, layer_config=layer_config)
+ ar.quantize()
+ with self.assertRaises(ValueError):
+ layer_config = {"model.decoder.layers.2.self_attn": {"bit": 2}} # should be bits
+ ar = AutoRound("facebook/opt-125m", scheme="W3A16", nsamples=1, iters=1, layer_config=layer_config)
+
if __name__ == "__main__":
unittest.main()
diff --git a/test/test_cpu/test_basic_usage.py b/test/test_cpu/test_basic_usage.py
index 02db860fe..ad9e9b56c 100644
--- a/test/test_cpu/test_basic_usage.py
+++ b/test/test_cpu/test_basic_usage.py
@@ -20,11 +20,10 @@ class TestAutoRoundCmd(unittest.TestCase):
python_path = sys.executable
##test llm script
- # res = os.system(
- # f"cd ../.. && {python_path} -m auto_round -h")
- # if res > 0 or res == -1:
- # assert False, "cmd line test fail, please have a check"
- #
+ res = os.system(f"cd ../.. && {python_path} -m auto_round -h")
+ if res > 0 or res == -1:
+ assert False, "cmd line test fail, please have a check"
+
res = os.system(
f"cd ../.. && {python_path} -m auto_round --model 'facebook/opt-125m' --seqlen 32 --iter 2 --nsamples 1 --format auto_gptq,auto_round --output_dir ./saved --tasks piqa"
)
diff --git a/test/test_cpu/test_export.py b/test/test_cpu/test_export.py
index 68942ac77..6d1cd8b46 100644
--- a/test/test_cpu/test_export.py
+++ b/test/test_cpu/test_export.py
@@ -25,7 +25,7 @@ class TestAutoRound(unittest.TestCase):
def setUpClass(self):
model_name = "facebook/opt-125m"
self.save_dir = "./saved"
- self.model = AutoModelForCausalLM.from_pretrained(model_name, torch_dtype="auto", trust_remote_code=True)
+ self.model = AutoModelForCausalLM.from_pretrained(model_name, dtype="auto", trust_remote_code=True)
self.tokenizer = AutoTokenizer.from_pretrained(model_name, trust_remote_code=True)
self.llm_dataloader = LLMDataLoader()
@@ -47,6 +47,7 @@ class TestAutoRound(unittest.TestCase):
seqlen=2,
dataset=self.llm_dataloader,
)
+
autoround.quantize()
quantized_model_path = "./saved"
autoround.save_quantized(output_dir=quantized_model_path, inplace=False, format="auto_gptq")
@@ -56,7 +57,7 @@ class TestAutoRound(unittest.TestCase):
continue
quantization_config = AutoRoundConfig()
model = AutoModelForCausalLM.from_pretrained(
- quantized_model_path, device_map="cpu", trust_remote_code=True, quantization_config=quantization_config
+ quantized_model_path, device_map="auto", trust_remote_code=True, quantization_config=quantization_config
)
tokenizer = AutoTokenizer.from_pretrained(quantized_model_path)
text = "There is a girl who likes adventure,"
diff --git a/test/test_cpu/test_gguf_format.py b/test/test_cpu/test_gguf_format.py
index d3f255397..61413c24b 100644
--- a/test/test_cpu/test_gguf_format.py
+++ b/test/test_cpu/test_gguf_format.py
@@ -242,10 +242,8 @@ class TestGGUF(unittest.TestCase):
"group_size": 32,
"sym": False,
"data_type": "int_asym_dq",
- "scale_dtype": torch.float32,
"super_bits": 6,
"super_group_size": 8,
- "in_blocks": False,
},
"model.embed_tokens": {"bits": 6, "group_size": 32, "super_bits": 6, "super_group_size": 8},
"model.layers.12.mlp.gate_proj": {"bits": 3},
diff --git a/test/test_cpu/test_scheme.py b/test/test_cpu/test_scheme.py
new file mode 100644
index 000000000..e36d2aa98
--- /dev/null
+++ b/test/test_cpu/test_scheme.py
@@ -0,0 +1,68 @@
+import shutil
+import sys
+import unittest
+
+from auto_round.schemes import QuantizationScheme
+
+sys.path.insert(0, "../..")
+
+from auto_round import AutoRound
+
+
+class TestAutoRound(unittest.TestCase):
+ @classmethod
+ def setUpClass(self):
+ self.model_name = "facebook/opt-125m"
+ self.save_folder = "./saved"
+
+ @classmethod
+ def tearDownClass(self):
+ shutil.rmtree(self.save_folder, ignore_errors=True)
+ shutil.rmtree("runs", ignore_errors=True)
+
+ def test_gguf(self):
+ ar = AutoRound("Qwen/Qwen3-0.6B", scheme="W2A16", nsamples=1, iters=1)
+ ar.quantize_and_save(self.save_folder, format="gguf:q4_k_m")
+ self.assertEqual(ar.bits, 4)
+ shutil.rmtree(self.save_folder, ignore_errors=True)
+
+ def test_w4a16(self):
+ ar = AutoRound(self.model_name, scheme="W4A16", nsamples=1, iters=1)
+ self.assertEqual(ar.bits, 4)
+ ar.quantize()
+
+ def test_w2a16_rtn(self):
+ ar = AutoRound(self.model_name, scheme="W2A16", nsamples=1, iters=0)
+ self.assertEqual(ar.bits, 2)
+ ar.quantize()
+
+ def test_mxfp4(self):
+ ar = AutoRound(self.model_name, scheme="MXFP4", nsamples=1, iters=1)
+ self.assertEqual(ar.bits, 4)
+ self.assertEqual(ar.act_bits, 4)
+ self.assertEqual(ar.data_type, "mx_fp")
+ self.assertEqual(ar.act_data_type, "mx_fp_rceil")
+ ar.quantize()
+
+ def test_scheme_in_layer_config(self):
+ layer_config = {
+ "model.decoder.layers.2.self_attn": {"bits": 2},
+ "model.decoder.layers.3.self_attn.v_proj": "W8A16",
+ "model.decoder.layers.4.self_attn.k_proj": QuantizationScheme.from_dict({"group_size": 64}),
+ }
+ ar = AutoRound(self.model_name, scheme="W3A16", nsamples=1, iters=1, layer_config=layer_config)
+
+ ar.quantize()
+ for n, m in ar.model.named_modules():
+ if n == "model.decoder.layers.2.self_attn.q_proj":
+ self.assertEqual(m.bits, 2)
+ if n == "model.decoder.layers.2.self_attn.k_proj":
+ self.assertEqual(m.bits, 2)
+ if n == "model.decoder.layers.3.self_attn.v_proj":
+ self.assertEqual(m.bits, 8)
+ if n == "model.decoder.layers.4.self_attn.k_proj":
+ self.assertEqual(m.group_size, 64)
+
+
+if __name__ == "__main__":
+ unittest.main()