PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度
[API CHANGE] Stage 1 add quant scheme and consolidate device and device_map (#774)
Wenhua Cheng8bcf7eab4bab
diff --git a/README.md b/README.mdindex 819167905..434ad93bb 100644--- a/README.md+++ b/README.md@@ -59,7 +59,7 @@ and [fbaldassarri](https://huggingface.co/fbaldassarri).Delivers strong performance even at 2–3 bits [example models](https://huggingface.co/collections/OPEA/2-3-bits-67a5f0bc6b49d73c01b4753b), with leading results at 4 bits [benchmark](https://huggingface.co/spaces/Intel/low_bit_open_llm_leaderboard).✅ **Ecosystem Integration**-Seamlessly works with **Transformers, vLLM,** TorchAO, sglang(on going,[pr](https://github.com/sgl-project/sglang/pull/6226)) and more.+Seamlessly works with **Transformers, vLLM,** and more.✅ **Multiple Formats Export**Support **AutoRound, AutoAWQ, AutoGPTQ, and GGUF** for maximum compatibility. Details are shown in [export formats](https://github.com/intel/auto-round/blob/main/docs/step_by_step.md#supported-export-formats)diff --git a/auto_round/autoround.py b/auto_round/autoround.pyindex f507d4b18..32213e570 100644--- a/auto_round/autoround.py+++ b/auto_round/autoround.py@@ -18,10 +18,12 @@ import reimport sysimport timeimport traceback+from dataclasses import asdictfrom typing import Any, Callable, Unionimport accelerateimport torch+from accelerate.big_modeling import dispatch_model, infer_auto_device_mapfrom torch import autocastfrom tqdm import tqdmfrom transformers import set_seed@@ -30,6 +32,7 @@ from auto_round.data_type import QUANT_FUNC_WITH_DTYPEfrom auto_round.data_type.utils import reshape_pad_tensor_by_group_sizefrom auto_round.export.export_to_gguf.config import GGUF_CONFIG, GGUF_INNER_CONFIG, ModelTypefrom auto_round.low_cpu_mem.utils import get_layers_before_block+from auto_round.schemes import PRESET_SCHEMES, QuantizationScheme, preset_name_to_schemefrom auto_round.sign_sgd import SignSGDfrom auto_round.special_model_handler import _handle_moe_modelfrom auto_round.utils import (@@ -63,6 +66,7 @@ from auto_round.utils import (get_layer_features,get_layer_names_in_block,get_lm_head_name,+ get_max_vram,get_module,get_quant_keys,get_shared_keys,@@ -107,37 +111,34 @@ class AutoRound(object):enable_torch_compile (bool): Whether to enable torch.compile for quant blocks/layers."""+ bits: int | None+ group_size: int | None+ sym: bool | None+ data_type: str | None+ act_bits: int | None+ act_group_size: int | None+ act_sym: bool | None+ act_data_type: str | None+ act_dynamic: bool | None+ super_bits: int | None+ super_group_size: int | None+def __init__(self,model: Union[torch.nn.Module, str],tokenizer=None,- bits: int = 4,- group_size: int = 128,- sym: bool = True,- layer_config: dict = None,- batch_size: int = 8,- amp: bool = True,- device: Union[str, torch.device, int] = 0,+ scheme: Union[str, dict, QuantizationScheme] = "W4A16",+ layer_config: dict[str, Union[str, dict, QuantizationScheme]] = None,dataset: Union[str, list, tuple, torch.utils.data.DataLoader] = "NeelNanda/pile-10k",- enable_minmax_tuning: bool = True,- lr: float = None,- minmax_lr: float = None,- low_gpu_mem_usage: bool = False,iters: int = 200,seqlen: int = 2048,nsamples: int = 128,- seed: int = 42,+ batch_size: int = 8,gradient_accumulate_steps: int = 1,- data_type: str = "int",- act_bits: int = 16,- act_group_size: int = None,- act_sym: bool = None,- act_data_type: str = None,- act_dynamic: bool = True,+ low_gpu_mem_usage: bool = False,+ device_map: Union[str, torch.device, int, dict] = 0,enable_torch_compile: bool = False,- device_map: Union[str, dict] = None,- disable_opt_rtn: bool = False,- enable_alg_ext: bool = False,+ seed: int = 42,**kwargs,):"""Initialize AutoRound with quantization and tuning configuration.@@ -145,6 +146,7 @@ class AutoRound(object):Args:model (torch.nn.Module | str): Model object or model name to load.tokenizer: Tokenizer for text processing. Required if `model` is not a string and `iters > 0`.+ scheme (str| dict | QuantizationScheme ): A preset scheme that defines the quantization configurationsbits (int, optional): Weight quantization bits. Defaults to 4.group_size (int, optional): Weight quantization group size. Defaults to 128.sym (bool, optional): Symmetric weight quantization. Defaults to True.@@ -173,7 +175,7 @@ class AutoRound(object):disable_opt_rtn (bool, optional): Disable RTN-mode optimization (iters=0). Defaults to False.enable_alg_ext (bool, optional): Enable algorithm extension (primarily for INT2). Defaults to False.**kwargs: Backward compatible options:- - lr_scheduler, sampler, not_use_best_mse, dynamic_max_gap,+ - enable_alg_ext, lr, lr_scheduler, sampler, not_use_best_mse, dynamic_max_gap,super_group_size, super_bits, scale_dtype ("fp16" etc.),nblocks, low_cpu_mem_usage, to_quant_block_names,enable_norm_bias_tuning, enable_quanted_input,@@ -198,14 +200,21 @@ class AutoRound(object):... # ...... }"""+ self.scheme = None+ self._parse_and_set_scheme(scheme, kwargs)+# Extra/legacy kwargs for backward compatibility# Major version releases may pack them with extra configuration options+ amp = kwargs.pop("amp", True)+ lr = kwargs.pop("lr", None)+ enable_alg_ext = kwargs.pop("enable_alg_ext", False)+ enable_minmax_tuning = kwargs.pop("enable_minmax_tuning", True)+ minmax_lr = kwargs.pop("minmax_lr", None)+ disable_opt_rtn = kwargs.pop("disable_opt_rtn", False)lr_scheduler = kwargs.pop("lr_scheduler", None)sampler = kwargs.pop("sampler", "rand")not_use_best_mse = kwargs.pop("not_use_best_mse", False)dynamic_max_gap = kwargs.pop("dynamic_max_gap", -1)- super_group_size = kwargs.pop("super_group_size", None)- super_bits = kwargs.pop("super_bits", None)scale_dtype = kwargs.pop("scale_dtype", "fp16")nblocks = kwargs.pop("nblocks", 1)low_cpu_mem_usage = kwargs.pop("low_cpu_mem_usage", False)@@ -214,16 +223,46 @@ class AutoRound(object):enable_quanted_input: bool = kwargs.pop("enable_quanted_input", True)disable_deterministic_algorithms = kwargs.pop("disable_deterministic_algorithms", False)static_kv_dtype = kwargs.pop("static_kv_dtype", None)+ device = kwargs.pop("device", None)+ if device is not None:+ logger.warning("`device` is deprecated, please use `device_map` instead")+self.vlm = kwargs.pop("vlm") if "vlm" in kwargs else Falseif kwargs:logger.warning(f"unrecognized keys {list(kwargs.keys())} were passed. Please check them.")- if device is not None and "," in str(device):+ if device_map is not None and "," in str(device_map):raise ValueError(- "API does not support explicit set multiple devices,"- " please set CUDA_VISIBLE_DEVICES yourself and use `device=auto` instead"+ "API does not support explicit set multiple devices," " please set CUDA_VISIBLE_DEVICES=0,1 yourself")+ if device_map is None:+ device_map = 0++ # Set device, must place after model loading+ if isinstance(device_map, (str, torch.device, int)):+ self.device = detect_device(device_map)+ elif isinstance(device_map, dict) and device_map:+ tmp_devices = []+ for val in device_map.values():+ if isinstance(val, (str, torch.device, int)): # could optimize+ tmp_device = detect_device(self.device_map)+ tmp_device = tmp_device.split(":")[0]+ tmp_devices.append(tmp_device)+ tmp_devices = list(set(tmp_devices))+ if len(tmp_devices) > 1:+ logger.warning(+ f"there are multiple device types in the device_map, "+ f"please make sure they are correct,use the first device {tmp_devices[0]} as the core device "+ )++ self.device = tmp_devices[0]++ if isinstance(device_map, dict) and device_map:+ self.device_map = device_map+ else:+ self.device_map = None+ self._set_device_map_in_blocks(self.device_map)if not disable_deterministic_algorithms:if "CUBLAS_WORKSPACE_CONFIG" not in os.environ:@@ -234,7 +273,9 @@ class AutoRound(object):self.quantized = Falseif isinstance(model, str):model, tokenizer, low_cpu_mem_usage = llm_load_model(- model, device=device, low_cpu_mem_mode=low_cpu_mem_usage+ model,+ device="cpu",+ low_cpu_mem_mode=low_cpu_mem_usage, # always load cpu first)elif tokenizer is None and iters > 0:raise ValueError("A tokenizer must be set for non-str model input")@@ -250,49 +291,33 @@ class AutoRound(object):self.tokenizer = tokenizerself.shared_cache_keys = get_shared_keys(self.model)- # Set quantization configs- self.bits = bits- self.data_type = data_type- tmp_bits = infer_bits_by_data_type(self.data_type)- if tmp_bits is not None and tmp_bits < 16 and tmp_bits != bits:- logger.warning(f"'data_type' do not match the specified 'bits' setting. Resetting 'bits' to {tmp_bits}.")- self.bits = tmp_bits- if tmp_bits is not None and tmp_bits < 16:- for supported_dtype in SUPPORTED_DTYPES: # to easily handle dtype mx_fp4 and layer_config={xxx:{bits:8}}- if data_type.startswith(supported_dtype):- if supported_dtype + str(tmp_bits) == self.data_type: # could not replace FP8_e4m3- self.data_type = supported_dtype- break- self.group_size = group_size- self.sym = sym+ # Some other quantization configsself.layer_config = {} if layer_config is None else layer_config- self.to_quant_block_names = to_quant_block_names+ # Check and convert to dict+ for key, item in self.layer_config.items():+ if isinstance(item, str):+ item = asdict(preset_name_to_scheme(item.upper()))+ self.layer_config[key] = item++ if isinstance(item, QuantizationScheme):+ config = asdict(item)+ tmp_keys = copy.deepcopy(list(config.keys()))+ for tmp_key in tmp_keys: ## Pop None value to be overridden+ if config[tmp_key] is None:+ config.pop(tmp_key)+ self.layer_config[key] = config+ elif isinstance(item, dict):+ item_keys = item.keys()+ expected_keys = list(QuantizationScheme.__annotations__.keys())+ if item_keys not in expected_keys:+ for item_key in item_keys:+ if item_key not in expected_keys:+ raise ValueError(+ f"the key {item_key} in layer_config for layer {key} is invalid,"+ f" only {expected_keys} are supported"+ )- # Activation- self.act_group_size = act_group_size if act_group_size is not None else group_size- self.act_bits = act_bits if act_bits is not None else self.bits- self.act_sym = act_sym if act_sym is not None else self.sym- self.act_dynamic = act_dynamic- self.act_data_type = act_data_type- if self.act_data_type is None:- if self.data_type in SUPPORTED_DTYPES and self.act_bits < 16:- self.act_data_type = data_type- logger.info(f"activation adopts {data_type}")- else:- self.act_data_type = "float"- tmp_act_bits = infer_bits_by_data_type(self.act_data_type)- if tmp_act_bits is not None and tmp_act_bits < 16 and tmp_act_bits != self.act_bits:- self.act_bits = tmp_act_bits- logger.warning(- f"`act_data_type` do not"- f" match the specified 'act_bits' setting. Resetting 'act_bits' to {tmp_act_bits}."- )- if tmp_act_bits is not None and tmp_act_bits < 16:- for supported_dtype in SUPPORTED_DTYPES: # to easily handle dtype mx_fp4 and layer_config={xxx:{bits:8}}- if self.act_data_type.startswith(supported_dtype):- if supported_dtype + str(tmp_act_bits) == self.act_data_type: # could not replace FP8_e4m3- self.act_data_type = supported_dtype- break+ self.to_quant_block_names = to_quant_block_names# Tuning hyperparametersself.seed = seed@@ -322,8 +347,6 @@ class AutoRound(object):self.dynamic_max_gap = dynamic_max_gapself.lr_scheduler = lr_schedulerself.optimizer = self._get_optimizer(None)- self.super_bits = super_bits- self.super_group_size = super_group_sizeself.disable_opt_rtn = disable_opt_rtnself.is_packing_immediate = False # whether to pack the layer immediately after tuning@@ -344,7 +367,7 @@ class AutoRound(object):if unsupport_meta_device(model):raise RuntimeError("AutoRound does not support parameters on meta device. "- "Please use more GPUs by setting `--device 0,1,2,3` or just place the model on CPU."+ "Please use more GPUs by setting `--device_map 0,1,2,3` or just place the model on CPU.")model = _handle_moe_model(model)self.model = model.eval()@@ -354,10 +377,6 @@ class AutoRound(object):all_blocks = get_block_names(model)self.quant_block_list = find_matching_blocks(model, all_blocks, self.to_quant_block_names)- # Set device, must place after model loading- self.device = detect_device(device) # must place after llm_load_model, because this one will convert auto- self.device_map = device_map- self._set_device_map_in_blocks(self.device_map)self.scale_dtype = convert_dtype_str2torch(scale_dtype)self._set_amp_dtype()self.cache_device = torch.device("cpu") if self.low_gpu_mem_usage else self.device@@ -416,6 +435,74 @@ class AutoRound(object):"super_group_size",]+ def _parse_and_set_scheme(self, scheme: Union[str, dict, QuantizationScheme], kwargs) -> None:+ """Parse and set the quantization scheme."""+ if isinstance(scheme, QuantizationScheme):+ scheme = asdict(scheme)+ elif isinstance(scheme, dict):+ scheme = scheme+ elif isinstance(scheme, str):+ scheme = scheme.upper()+ self.scheme = scheme+ scheme = asdict(preset_name_to_scheme(scheme))++ scheme_keys = (+ "bits",+ "group_size",+ "sym",+ "data_type",+ "act_bits",+ "act_group_size",+ "act_sym",+ "act_data_type",+ "act_dynamic",+ "super_bits",+ "super_group_size",+ )+ for key in scheme_keys:+ if key in kwargs and kwargs[key] is not None:+ setattr(self, key, kwargs[key])+ else:+ setattr(self, key, scheme.get(key, None))+ kwargs.pop(key, None)+ if self.act_dynamic is None:+ self.act_dynamic = True++ tmp_bits = infer_bits_by_data_type(self.data_type)+ if tmp_bits is not None and tmp_bits < 16 and tmp_bits != self.bits:+ logger.warning(f"'data_type' do not match the specified 'bits' setting. Resetting 'bits' to {tmp_bits}.")+ self.bits = tmp_bits+ if tmp_bits is not None and tmp_bits < 16:+ for supported_dtype in SUPPORTED_DTYPES: # to easily handle dtype mx_fp4 and layer_config={xxx:{bits:8}}+ if self.data_type.startswith(supported_dtype):+ if supported_dtype + str(tmp_bits) == self.data_type: # could not replace FP8_e4m3+ self.data_type = supported_dtype+ break++ self.act_group_size = self.act_group_size if self.act_group_size is not None else self.group_size+ self.act_bits = self.act_bits if self.act_bits is not None else 16+ self.act_sym = self.act_sym if self.act_sym is not None else self.sym++ if self.act_data_type is None:+ if self.data_type in SUPPORTED_DTYPES and self.act_bits < 16:+ self.act_data_type = self.data_type+ logger.info(f"activation adopts {self.data_type}")+ else:+ self.act_data_type = "float"+ tmp_act_bits = infer_bits_by_data_type(self.act_data_type)+ if tmp_act_bits is not None and tmp_act_bits < 16 and tmp_act_bits != self.act_bits:+ self.act_bits = tmp_act_bits+ logger.warning(+ f"`act_data_type` do not"+ f" match the specified 'act_bits' setting. Resetting 'act_bits' to {tmp_act_bits}."+ )+ if tmp_act_bits is not None and tmp_act_bits < 16:+ for supported_dtype in SUPPORTED_DTYPES: # to easily handle dtype mx_fp4 and layer_config={xxx:{bits:8}}+ if self.act_data_type.startswith(supported_dtype):+ if supported_dtype + str(tmp_act_bits) == self.act_data_type: # could not replace FP8_e4m3+ self.act_data_type = supported_dtype+ break+def _adjust_torch_compile(self, enable_torch_compile: bool) -> None:"""Sets the torch compile configuration for the tuning."""self.enable_torch_compile = enable_torch_compile@@ -976,8 +1063,6 @@ class AutoRound(object):# Dispatch multi-GPU model if necessaryif hasattr(model, "hf_device_map") and len(model.hf_device_map) > 1:- from accelerate.big_modeling import dispatch_model-dispatch_model(model, model.hf_device_map)def register_act_hook(model):@@ -1007,8 +1092,6 @@ class AutoRound(object):try:# Move model to target deviceif hasattr(self.model, "hf_device_map") and len(self.model.hf_device_map) > 1:- from accelerate.big_modeling import dispatch_model-dispatch_model(self.model, self.model.hf_device_map)else:model = model.to(self.device)@@ -1099,20 +1182,26 @@ class AutoRound(object):Raises:NotImplementedError: If multiple non-fake GGUF formats are specified."""- if not hasattr(self, "formats"):+ gguf_scheme = False+ if isinstance(self.scheme, str) and "gguf" in self.scheme.lower():+ gguf_scheme = True++ if not hasattr(self, "formats") and not gguf_scheme:return False- has_gguf: bool = any("gguf" in fmt for fmt in self.formats)+ has_gguf: bool = gguf_scheme or any("gguf" in fmt for fmt in self.formats)if not has_gguf:return False+ if hasattr(self, "formats"):+ formats: list[str] = [fmt for fmt in self.formats if "fake" not in fmt]+ if not (len(formats) == 1 and "gguf" in formats[0]):+ raise NotImplementedError("Only one GGUF format can be set at a time.")+ target_format: str = formats[0]- formats: list[str] = [fmt for fmt in self.formats if "fake" not in fmt]- if not (len(formats) == 1 and "gguf" in formats[0]):- raise NotImplementedError("Only one GGUF format can be set at a time.")+ else:+ target_format = self.scheme.lower()- target_format: str = formats[0]tie_word_embeddings: bool = getattr(getattr(self.model, "config", None), "tie_word_embeddings", True)-for name, module in self.model.named_modules():if isinstance(module, torch.nn.Embedding):key: str = "lm_head" if tie_word_embeddings else "embedding"@@ -1301,7 +1390,7 @@ class AutoRound(object):if has_gguf_k and not self.disable_opt_rtn:self._quant_rtn_with_imatrix(all_to_quantized_module_names)elif self.act_bits <= 8 and check_need_act_calibration(- self.act_dynamic, self.act_data_type+ self.act_dynamic, self.act_data_type, self.act_bits): # TODO, mixed datatype has bughook_handles = self._register_act_max_hook(self.model)try:@@ -1407,7 +1496,6 @@ class AutoRound(object):convert_fp8_model_to_16b_model(block, dtype=self.amp_dtype)# Dispatch model if neededif self.device_map is not None:- from accelerate import dispatch_modelfrom accelerate.hooks import AlignDevicesHook, add_hook_to_modulefor _, m in block.named_modules():@@ -1669,8 +1757,6 @@ class AutoRound(object):enable_quanted_input = Falseif hasattr(self.model, "hf_device_map") and len(self.model.hf_device_map) > 1 and enable_quanted_input:- from accelerate.big_modeling import dispatch_model-dispatch_model(self.model, self.model.hf_device_map)if enable_quanted_input:@@ -2005,31 +2091,46 @@ class AutoRound(object):layer_names = []if self.low_gpu_mem_usage or (- str(self.model.device == "cpu")- and len(block_names) == 1+ len(block_names) == 1and len(layer_names) == 0and not self.has_qlayer_outside_blockand (last_cache_name is None or last_cache_name in block_names)):- ## low_gpu_mem_usage or calibrate only the embedding layer, which is also very fast on CPU+ # low_gpu_mem_usage or calibrate only the embedding layer, which is also very fast on CPUall_inputs = self.cache_inter_data(block_names, nsamples, layer_names=[], last_cache_name=last_cache_name)else:try:if not self.model.device.type == "meta":if hasattr(self.model, "hf_device_map") and len(self.model.hf_device_map) > 1:- pass+ self.model = dispatch_model(self.model, device_map=self.model.hf_device_map)else:- self.model = self.model.to(self.device)+ # Change this if new device is supported+ if str(self.model.device) == "cpu" and (+ self.device.startswith("xpu") or self.device.startswith("cuda")+ ):+ max_memory = get_max_vram() # TODO model is not evenly split+ no_split_modules = getattr(self.model, "_no_split_modules", [])+ device_map = infer_auto_device_map(+ self.model, max_memory=max_memory, no_split_module_classes=no_split_modules+ )++ self.model = dispatch_model(self.model, device_map=device_map)+ else:+ self.model = self.model.to(self.device)+all_inputs = self.cache_inter_data(block_names, nsamples, layer_names=layer_names, last_cache_name=last_cache_name)+ if hasattr(self.model, "hf_device_map") and len(self.model.hf_device_map) > 1:+ accelerate.hooks.remove_hook_from_submodules(self.model)+except RuntimeError as e:cuda_error_msg = traceback.format_exc()try:logger.info("switch to cpu to cache block inputs")if self.has_qlayer_outside_block or self.__class__.__name__ == "AutoRoundMLLM":logger.warning(- "we strongly recommend using more GPUs in calibration."+ "we recommend using more GPUs in calibration."" Otherwise, some layers may fall back to `rtn` mode, which can affect accuracy.")if hasattr(self.model, "hf_device_map") and len(self.model.hf_device_map) > 1:@@ -2308,10 +2409,8 @@ class AutoRound(object):nsamples = len(inputs)last_best_iter = 0best_loss = torch.finfo(torch.float).max- mse_loss = torch.nn.MSELoss().to(device)scaler = self._get_scaler() # pylint: disable=assignment-from-noneinit_loss = None- # best_v, best_min_scale, best_max_scale = torch.tensor(0), torch.tensor(1.0), torch.tensor(1.0)gradient_accumulate_steps = self.batch_size ##Force to low gpubatch_size = 1 ##Force to low gpupick_samples = batch_size * gradient_accumulate_steps@@ -2412,7 +2511,7 @@ class AutoRound(object):for n, m in model.named_modules():if (hasattr(m, "act_dynamic")- and check_need_act_calibration(m.act_dynamic, m.act_data_type)+ and check_need_act_calibration(m.act_dynamic, m.act_data_type, m.act_bits)and check_to_quantized(m)):hook = m.register_forward_hook(get_act_max_hook)@@ -2424,9 +2523,10 @@ class AutoRound(object):config = self.layer_config[n]act_dynamic = config.get("act_dynamic", True)act_data_type = config.get("act_data_type", None)+ act_bits = config.get("act_data_type", 16)if (config["bits"] <= 8- and check_need_act_calibration(act_dynamic, act_data_type)+ and check_need_act_calibration(act_dynamic, act_data_type, act_bits)and check_to_quantized(config)):hook = m.register_forward_hook(get_act_max_hook)@@ -2461,8 +2561,6 @@ class AutoRound(object):set_module(block, n, new_layer)if self.device_map is not None:- from accelerate import dispatch_model-for n, m in block.named_modules():if len(list(m.children())) != 0 or not hasattr(m, "tuning_device"):continue@@ -3054,6 +3152,7 @@ class AutoRoundAdam(AutoRound):Args:model: The PyTorch model to be quantized.tokenizer: An optional tokenizer for processing input data.+ scheme (str| dict | QuantizationScheme ): A preset scheme that defines the quantization configurationsbits (int): Number of bits for quantization (default is 4).group_size (int): Size of the quantization group (default is 128).sym (bool): Whether sym to be used (default is True).@@ -3097,92 +3196,52 @@ class AutoRoundAdam(AutoRound):The quantized model."""+ bits: int | None+ group_size: int | None+ sym: bool | None+ data_type: str | None+ act_bits: int | None+ act_group_size: int | None+ act_sym: bool | None+ act_data_type: str | None+ act_dynamic: bool | None+ super_bits: int | None+ super_group_size: int | None+def __init__(self,model: Union[torch.nn.Module, str],tokenizer=None,- bits: int = 4,- group_size: int = 128,- sym: bool = True,- layer_config=None,- batch_size: int = 8,- amp: bool = True,- device: Union[str, torch.device, int] = 0,- lr_scheduler=None,+ scheme: Union[str, dict, QuantizationScheme] = "W4A16",+ layer_config: dict[str, Union[str, dict, QuantizationScheme]] = None,dataset: Union[str, list, tuple, torch.utils.data.DataLoader] = "NeelNanda/pile-10k",- enable_quanted_input: bool = True,- enable_minmax_tuning: bool = True,- lr: float = None,- minmax_lr: float = None,- low_gpu_mem_usage: bool = False,- low_cpu_mem_usage: int = 0,iters: int = 200,seqlen: int = 2048,nsamples: int = 128,- sampler: str = "rand",- seed: int = 42,- nblocks: int = 1,+ batch_size: int = 8,gradient_accumulate_steps: int = 1,- not_use_best_mse: bool = False,- dynamic_max_gap: int = -1,- data_type: str = "int",- scale_dtype: str = "fp16",- act_bits: int = 16,- act_group_size: int = None,- act_sym: bool = None,- act_data_type: str = None,- act_dynamic: bool = True,- to_quant_block_names: Union[str, list] = None,- enable_norm_bias_tuning: bool = False,+ low_gpu_mem_usage: bool = False,+ device_map: Union[str, int, torch.device, dict] = 0,enable_torch_compile: bool = False,- device_map: Union[str, dict] = None,+ seed: int = 42,optimizer="AdamW",- super_bits: int = None,- super_group_size: int = None,- disable_opt_rtn: bool = False,**kwargs,):super(AutoRoundAdam, self).__init__(model=model,tokenizer=tokenizer,- bits=bits,- group_size=group_size,- sym=sym,+ scheme=scheme,layer_config=layer_config,batch_size=batch_size,- amp=amp,- device=device,- lr_scheduler=lr_scheduler,dataset=dataset,- enable_quanted_input=enable_quanted_input,- enable_minmax_tuning=enable_minmax_tuning,- lr=lr,- minmax_lr=minmax_lr,low_gpu_mem_usage=low_gpu_mem_usage,- low_cpu_mem_usage=low_cpu_mem_usage,iters=iters,seqlen=seqlen,nsamples=nsamples,- sampler=sampler,seed=seed,- nblocks=nblocks,gradient_accumulate_steps=gradient_accumulate_steps,- not_use_best_mse=not_use_best_mse,- dynamic_max_gap=dynamic_max_gap,- data_type=data_type,- scale_dtype=scale_dtype,- act_bits=act_bits,- act_group_size=act_group_size,- act_sym=act_sym,- act_data_type=act_data_type,- act_dynamic=act_dynamic,- to_quant_block_names=to_quant_block_names,- enable_norm_bias_tuning=enable_norm_bias_tuning,enable_torch_compile=enable_torch_compile,device_map=device_map,- super_bits=super_bits,- super_group_size=super_group_size,- disable_opt_rtn=disable_opt_rtn,**kwargs,)diff --git a/auto_round/mllm/eval.py b/auto_round/mllm/eval.pyindex 4d3f5e8e8..e08c88ebd 100644--- a/auto_round/mllm/eval.py+++ b/auto_round/mllm/eval.py@@ -52,9 +52,8 @@ import timefrom functools import partialimport numpy as np-import pandas as pd-from ..utils import LazyImport, logger+from auto_round.utils import LazyImport, loggervlmeval = LazyImport("vlmeval")@@ -304,6 +303,8 @@ def mllm_eval(if mode == "all":eval_results = dataset.evaluate(result_file, **judge_kwargs)+ import pandas as pd+if eval_results is not None:if not isinstance(eval_results, dict) and not isinstance(eval_results, pd.DataFrame):raise TypeError("Unsupported eval result type")diff --git a/auto_round/schemes.py b/auto_round/schemes.pynew file mode 100644index 000000000..e556acd29--- /dev/null+++ b/auto_round/schemes.py@@ -0,0 +1,175 @@+# Copyright (c) 2025 Intel Corporation+#+# Licensed under the Apache License, Version 2.0 (the "License");+# you may not use this file except in compliance with the License.+# You may obtain a copy of the License at+#+# http://www.apache.org/licenses/LICENSE-2.0+#+# Unless required by applicable law or agreed to in writing, software+# distributed under the License is distributed on an "AS IS" BASIS,+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+# See the License for the specific language governing permissions and+# limitations under the License.+import copy+from copy import deepcopy+from dataclasses import dataclass+from typing import Optional++__all__ = ["QuantizationScheme", "preset_name_to_scheme"]+++@dataclass+class QuantizationScheme:+ bits: int = 4+ group_size: int = 128+ sym: bool = True+ data_type: str = "int"+ act_bits: Optional[int] = None+ act_group_size: Optional[int] = None+ act_sym: Optional[bool] = None+ act_data_type: Optional[str] = None+ act_dynamic: Optional[bool] = None+ super_bits: Optional[int] = None+ super_group_size: Optional[int] = None++ @classmethod+ def from_dict(cls, config: dict):+ return cls(**config)+++def preset_name_to_scheme(name: str) -> QuantizationScheme:+ """Get a QuantizationScheme instance from a preset scheme name."""+ name = name.upper()++ if name not in PRESET_SCHEMES:+ raise KeyError(f"Unknown preset scheme name {name}, " f"available names: {list(PRESET_SCHEMES.keys())}")++ scheme_args = deepcopy(PRESET_SCHEMES[name])+ return scheme_args+++def is_preset_scheme(name: str) -> bool:+ """Check if the given name is a preset scheme name."""+ return name.upper() in PRESET_SCHEMES+++W4A16 = QuantizationScheme.from_dict(+ {+ "bits": 4,+ "sym": True,+ "group_size": 128,+ "data_type": "int",+ "act_bits": 16,+ }+)++W2A16 = QuantizationScheme.from_dict(+ {+ "bits": 2,+ "sym": True,+ "group_size": 128,+ "data_type": "int",+ "act_bits": 16,+ }+)++W3A16 = QuantizationScheme.from_dict(+ {+ "bits": 3,+ "sym": True,+ "group_size": 128,+ "data_type": "int",+ "act_bits": 16,+ }+)++W8A16 = QuantizationScheme.from_dict(+ {+ "bits": 8,+ "sym": True,+ "group_size": 128,+ "data_type": "int",+ "act_bits": 16,+ }+)++MXFP4 = QuantizationScheme.from_dict(+ {+ "bits": 4,+ "group_size": 32,+ "data_type": "mx_fp",+ "act_bits": 4,+ "act_data_type": "mx_fp_rceil",+ }+)++MXFP8 = QuantizationScheme.from_dict(+ {+ "bits": 8,+ "group_size": 32,+ "data_type": "mx_fp",+ "act_bits": 8,+ "act_data_type": "mx_fp_rceil",+ }+)++NVFP4 = QuantizationScheme.from_dict(+ {+ "bits": 4,+ "group_size": 16,+ "data_type": "nv_fp",+ "act_bits": 4,+ "act_data_type": "nv_fp_with_static_gs",+ }+)++FPW8A16 = QuantizationScheme.from_dict(+ {+ "bits": 8,+ "group_size": 0,+ "data_type": "fp",+ "act_bits": 16,+ "act_data_type": "fp",+ }+)++# FP8 = asdict(QuantArgs.from_dict({+# "bits": 8,+# "group_size": 128,+# "data_type": "fp",+# "act_bits": 8,+# "act_data_type": "fp",+# }))++FPW8_STATIC = QuantizationScheme.from_dict(+ {+ "bits": 8,+ "group_size": -1,+ "data_type": "fp",+ "act_bits": 8,+ "act_group_size": 0,+ "act_data_type": "fp",+ "act_dynamic": False,+ }+)++PRESET_SCHEMES = {+ "W4A16": W4A16,+ "W2A16": W2A16,+ "W3A16": W3A16,+ "W8A16": W8A16,+ "MXFP4": MXFP4,+ "MXFP8": MXFP8,+ "NVFP4": NVFP4,+ "FPW8A16": FPW8A16,+ "FPW8_STATIC": FPW8_STATIC,+}+from auto_round.export.export_to_gguf.config import GGUF_CONFIG++for key, val in GGUF_CONFIG.items():+ value = copy.deepcopy(val)+ value.pop("mostly", None)+ value.pop("embedding", None)+ value.pop("lm_head", None)+ PRESET_SCHEMES[key.upper()] = QuantizationScheme.from_dict(value)diff --git a/auto_round/script/llm.py b/auto_round/script/llm.pyindex 3caefcb85..0aaadef83 100644--- a/auto_round/script/llm.py+++ b/auto_round/script/llm.py@@ -32,6 +32,7 @@ import osimport reimport sys+from auto_round.schemes import PRESET_SCHEMESfrom auto_round.utils import (clear_memory,get_device_and_parallelism,@@ -39,7 +40,6 @@ from auto_round.utils import (get_model_dtype,infer_bits_by_data_type,set_cuda_visible_devices,- str2bool,)@@ -55,11 +55,35 @@ class BasicArgumentParser(argparse.ArgumentParser):self.add_argument("--eval", action="store_true", help="whether to use eval only mode")- self.add_argument("--bits", default=4, type=int, help="number of weight bits")+ self.add_argument(+ "--scheme",+ default="W4A16",+ type=str,+ # choices=["W4A16", "W2A16", "W3A16", "W8A16", "MXFP4", "MXFP8", "NVFP4", "FPW8A16", "FPW8_STATIC"],+ help="quantization scheme",+ )++ self.add_argument("--bits", default=None, type=int, help="number of weight bits")+ self.add_argument("--group_size", default=None, type=int, help="group size")+ self.add_argument("--asym", action="store_true", help="whether to use asym quantization")+ self.add_argument("--data_type", "--dtype", default=None, help="data type for tuning, 'int', 'mx_fp' and etc")+ self.add_argument("--act_bits", default=None, type=int, help="activation bits")+ self.add_argument("--act_group_size", default=None, type=int, help="activation group size")+ self.add_argument(+ "--super_group_size", default=None, type=int, help="the number of super group size when use double quant."+ )++ self.add_argument(+ "--super_bits", default=None, type=int, help="number of scale and mins quant bits for double quant."+ )+ self.add_argument("--act_data_type", "--act_dtype", default=None, type=str, help="activation data type")++ self.add_argument("--disable_act_dynamic", action="store_true", help="activation static quantization")self.add_argument("--eval_bs", default=None, type=int, help="batch size in evaluation")self.add_argument(+ "--device_map","--device","--devices",default="0",@@ -71,8 +95,6 @@ class BasicArgumentParser(argparse.ArgumentParser):"set --device 0,1,2 to use multiple cards.",)- self.add_argument("--asym", action="store_true", help="whether to use asym quantization")-self.add_argument("--dataset", default="NeelNanda/pile-10k", type=str, help="the dataset for quantization training")@@ -96,8 +118,6 @@ class BasicArgumentParser(argparse.ArgumentParser):self.add_argument("--format", default="auto_round", type=str, help="the format to save the model")- self.add_argument("--data_type", "--dtype", default="int", help="data type for tuning, 'int', 'mx_fp' and etc")-self.add_argument("--scale_dtype",default="fp16",@@ -175,10 +195,6 @@ class BasicArgumentParser(argparse.ArgumentParser):help="force to convert the dtype, some backends supports fp16 dtype better",)- self.add_argument("--act_bits", default=16, type=int, help="activation bits")-- self.add_argument("--act_group_size", default=None, type=int, help="activation group size")-self.add_argument("--fp_layers", default="", type=str, help="list of Layer names to maintain original data type")@@ -200,24 +216,10 @@ class BasicArgumentParser(argparse.ArgumentParser):self.add_argument("--enable_alg_ext", action="store_true", help="whether to enable probably better algorithm")- self.add_argument("--act_data_type", "--act_dtype", default=None, type=str, help="activation data type")-- self.add_argument("--disable_act_dynamic", action="store_true", help="activation static quantization")-self.add_argument("--disable_deterministic_algorithms", action="store_true", help="disable torch deterministic algorithms.")- self.add_argument("--device_map", default=None, type=str, help="device_map for block in tuning phase")-- self.add_argument(- "--super_group_size", default=None, type=int, help="the number of super group size when use double quant."- )-- self.add_argument(- "--super_bits", default=None, type=int, help="number of scale and mins quant bits for double quant."- )-self.add_argument("--eval_model_dtype", default=None, type=str, help="the torch_dytpe to load the model for evaluation.")@@ -263,6 +265,7 @@ class EvalArgumentParser(argparse.ArgumentParser):)self.add_argument("--mllm", action="store_true", help="whether to eval multi-modal model.")self.add_argument(+ "--device_map","--device","--devices",default="0",@@ -294,8 +297,6 @@ class EvalArgumentParser(argparse.ArgumentParser):def setup_parser():parser = BasicArgumentParser()- parser.add_argument("--group_size", default=128, type=int, help="group size")-parser.add_argument("--batch_size", "--train_bs", "--bs", default=8, type=int, help="train batch size")parser.add_argument("--iters", "--iter", default=200, type=int, help="iteration to tune each block")@@ -317,8 +318,6 @@ def setup_parser():def setup_best_parser():parser = BasicArgumentParser()- parser.add_argument("--group_size", default=128, type=int, help="group size")-parser.add_argument("--batch_size", "--train_bs", "--bs", default=8, type=int, help="train batch size")parser.add_argument("--iters", "--iter", default=1000, type=int, help="iterations to tune each block")@@ -342,8 +341,6 @@ def setup_best_parser():def setup_light_parser():parser = BasicArgumentParser()- parser.add_argument("--group_size", default=128, type=int, help="group size")-parser.add_argument("--batch_size", "--train_bs", "--bs", default=8, type=int, help="train batch size")parser.add_argument("--iters", "--iter", default=50, type=int, help="iterations to tune each block")@@ -366,8 +363,6 @@ def setup_light_parser():def setup_fast_parser():parser = BasicArgumentParser()- parser.add_argument("--group_size", default=128, type=int, help="group size")-parser.add_argument("--batch_size", "--train_bs", "--bs", default=4, type=int, help="train batch size")parser.add_argument("--iters", default=200, type=int, help="iterations to tune each block")@@ -399,9 +394,7 @@ def tune(args):if args.eval_bs is None:args.eval_bs = "auto"-- import transformers- from transformers import AutoConfig, AutoModel, AutoModelForCausalLM, AutoTokenizer+ from transformers import AutoConfig, AutoModelForCausalLM, AutoTokenizerfrom auto_round.utils import detect_device, get_library_version, logger@@ -424,9 +417,9 @@ def tune(args):if "marlin" in args.format and args.asym is True:raise RuntimeError("marlin backend only supports sym quantization, please remove --asym")- ##must set this before import torch- set_cuda_visible_devices(args.device)- device_str, use_auto_mapping = get_device_and_parallelism(args.device)+ # Must set this before import torch+ set_cuda_visible_devices(args.device_map)+ device_str, use_auto_mapping = get_device_and_parallelism(args.device_map)import torch@@ -446,8 +439,8 @@ def tune(args):if args.mllm:model, processor, tokenizer, image_processor = mllm_load_model(model_name,- device=device_str,- use_auto_mapping=use_auto_mapping,+ device="cpu",+ use_auto_mapping=False,trust_remote_code=not args.disable_trust_remote_code,model_dtype=args.model_dtype,)@@ -455,9 +448,9 @@ def tune(args):else:model, tokenizer, low_cpu_mem_usage = llm_load_model(model_name,- use_auto_mapping=use_auto_mapping,+ use_auto_mapping=False,trust_remote_code=not args.disable_trust_remote_code,- device=device_str,+ device="cpu",low_cpu_mem_mode=args.low_cpu_mem_mode,low_cpu_mem_tmp_dir=args.low_cpu_mem_tmp_dir,model_dtype=args.model_dtype,@@ -495,7 +488,7 @@ def tune(args):and "awq" not in formatand "llmcompressor" not in format):- ##TODO gptq could support some mixed precision config+ # TODO gptq could support some mixed precision configlogger.warning(f"mixed precision exporting does not support {format} currently")lm_head_layer_name = "lm_head"@@ -506,7 +499,7 @@ def tune(args):if config.tie_word_embeddings and hasattr(model, "_tied_weights_keys"):tied_keys = model._tied_weights_keysfor item in tied_keys:- if lm_head_layer_name in item: ##TODO extend to encoder-decoder layer, seq classification model+ if lm_head_layer_name in item: # TODO extend to encoder-decoder layer, seq classification modelargs.quant_lm_head = Falselogger.warning("reset `quant_lm_head` to `False` as quantizing lm_head with tied weights has not been "@@ -533,13 +526,22 @@ def tune(args):logger.warning(f"The AutoAWQ format may not be supported due to {info}")enable_torch_compile = True if "--enable_torch_compile" in sys.argv else False-+ sym = None # the default value should be None now+ if args.asym: # if the scheme is asym, how to set it to sym is an issue+ sym = False+ act_dynamic = None+ if args.disable_act_dynamic:+ act_dynamic = False+ scheme = args.scheme.upper()+ if scheme not in PRESET_SCHEMES:+ raise ValueError(f"{scheme} is not supported. only {PRESET_SCHEMES.keys()} are supported ")autoround = round(model=model,tokenizer=tokenizer,+ scheme=scheme,bits=args.bits,group_size=args.group_size,- sym=not args.asym,+ sym=sym,batch_size=args.batch_size,dataset=args.dataset,seqlen=args.seqlen,@@ -548,7 +550,6 @@ def tune(args):lr=args.lr,minmax_lr=args.minmax_lr,enable_quanted_input=not args.disable_quanted_input,- device=device_str,amp=not args.disable_amp,nsamples=args.nsamples,seed=args.seed,@@ -566,7 +567,7 @@ def tune(args):to_quant_block_names=args.to_quant_block_names,enable_torch_compile=enable_torch_compile,act_data_type=args.act_data_type,- act_dynamic=not args.disable_act_dynamic,+ act_dynamic=act_dynamic,device_map=args.device_map,super_group_size=args.super_group_size,super_bits=args.super_bits,@@ -723,7 +724,7 @@ def tune(args):from auto_round.eval.evaluation import simple_evaluatetasks, model_args, device_str = _eval_init(- args.tasks, eval_folder, args.device, args.disable_trust_remote_code, dtype=eval_model_dtype+ args.tasks, eval_folder, args.device_map, args.disable_trust_remote_code, dtype=eval_model_dtype)st = time.time()if "llama" in args.model.lower():@@ -753,7 +754,7 @@ def eval(args):import timetasks, model_args, device_str = _eval_init(- args.tasks, args.model, args.device, args.disable_trust_remote_code, args.eval_model_dtype+ args.tasks, args.model, args.device_map, args.disable_trust_remote_code, args.eval_model_dtype)# load after _eval_int in order to make sure import torch after set CUDA_VISIBLE_DEVICESdiff --git a/auto_round/script/mllm.py b/auto_round/script/mllm.pyindex 1f55402df..6ab5a264f 100644--- a/auto_round/script/mllm.py+++ b/auto_round/script/mllm.py@@ -16,6 +16,7 @@ import argparseimport osimport sys+from auto_round.schemes import PRESET_SCHEMESfrom auto_round.utils import (clear_memory,get_device_and_parallelism,@@ -40,7 +41,18 @@ class BasicArgumentParser(argparse.ArgumentParser):self.add_argument("--eval", action="store_true", help="whether to use eval only mode.")- self.add_argument("--bits", default=4, type=int, help="weight bits")+ self.add_argument(+ "--scheme",+ default="W4A16",+ type=str,+ help="quantization scheme",+ )++ self.add_argument("--bits", default=None, type=int, help="number of weight bits")+ self.add_argument("--group_size", default=None, type=int, help="group size")+ self.add_argument("--asym", action="store_true", help="whether to use asym quantization")+ self.add_argument("--data_type", "--dtype", default=None, help="data type for tuning, 'int', 'mx_fp' and etc")+ self.add_argument("--act_bits", default=None, type=int, help="activation bits")self.add_argument("--eval_bs", default=None, type=int, help="batch size in evaluation")@@ -56,8 +68,6 @@ class BasicArgumentParser(argparse.ArgumentParser):"set --device 0,1,2 to use multiple cards.",)- self.add_argument("--asym", action="store_true", help="whether to use asym quantization")-self.add_argument("--dataset",type=str,@@ -91,8 +101,6 @@ class BasicArgumentParser(argparse.ArgumentParser):self.add_argument("--format", default="auto_round", type=str, help="the format to save the model")- self.add_argument("--data_type", "--dtype", default="int", help="data type for tuning, 'int', 'mx_fp' and etc")-self.add_argument("--scale_dtype",default="fp16",@@ -154,8 +162,6 @@ class BasicArgumentParser(argparse.ArgumentParser):help="force to convert the dtype, some backends supports fp16 dtype better",)- self.add_argument("--act_bits", default=16, type=int, help="activation bits")-self.add_argument("--fp_layers", default="", type=str, help="layers to maintain original data type")self.add_argument(@@ -221,8 +227,6 @@ class BasicArgumentParser(argparse.ArgumentParser):def setup_parser():parser = BasicArgumentParser()- parser.add_argument("--group_size", default=128, type=int, help="group size")-parser.add_argument("--batch_size", "--train_bs", "--bs", default=8, type=int, help="train batch size")parser.add_argument("--iters", "--iter", default=200, type=int, help=" iters")@@ -323,9 +327,9 @@ def tune(args):if format not in SUPPORTED_FORMATS:raise ValueError(f"{format} is not supported, we only support {SUPPORTED_FORMATS}")- ##must set this before import torch- set_cuda_visible_devices(args.device)- device_str, use_auto_mapping = get_device_and_parallelism(args.device)+ # Must set this before import torch+ set_cuda_visible_devices(args.device_map)+ device_str, use_auto_mapping = get_device_and_parallelism(args.device_map)import torch@@ -342,8 +346,9 @@ def tune(args):model, processor, tokenizer, image_processor = mllm_load_model(model_name,+ device="cpu",torch_dtype=torch_dtype,- use_auto_mapping=use_auto_mapping,+ use_auto_mapping=False,trust_remote_code=not args.disable_trust_remote_code,model_dtype=args.model_dtype,)@@ -438,16 +443,26 @@ def tune(args):"trust_remote_code": not args.disable_trust_remote_code,"model_dtype": args.model_dtype,}++ sym = None # the default value should be None now+ if args.asym: # if the scheme is asym, how to set it to sym is an issue+ sym = False++ scheme = args.scheme.upper()+ if scheme not in PRESET_SCHEMES:+ raise ValueError(f"{scheme} is not supported. only {PRESET_SCHEMES.keys()} are supported ")+autoround = round(model,tokenizer,+ scheme=args.scheme,processor=processor,image_processor=image_processor,dataset=args.dataset,extra_data_dir=args.extra_data_dir,bits=args.bits,group_size=args.group_size,- sym=not args.asym,+ sym=sym,batch_size=args.batch_size,seqlen=seqlen,nblocks=args.nblocks,@@ -459,7 +474,6 @@ def tune(args):truncation=args.truncation,nsamples=args.nsamples,low_gpu_mem_usage=args.low_gpu_mem_usage,- device=device_str,seed=args.seed,gradient_accumulate_steps=args.gradient_accumulate_steps,scale_dtype=args.scale_dtype,@@ -503,8 +517,8 @@ def tune(args):def vlmeval(args):- set_cuda_visible_devices(args.device)- device_str, parallelism = get_device_and_parallelism(args.device)+ set_cuda_visible_devices(args.device_map)+ device_str, parallelism = get_device_and_parallelism(args.device_map)if parallelism:os.environ["AUTO_SPLIT"] = "1"if isinstance(args.tasks, str):@@ -582,8 +596,8 @@ def setup_lmms_parser():def lmms_eval(args):- set_cuda_visible_devices(args.device)- device_str, parallelism = get_device_and_parallelism(args.device)+ set_cuda_visible_devices(args.device_map)+ device_str, parallelism = get_device_and_parallelism(args.device_map)from auto_round.mllm import lmms_evaldiff --git a/auto_round/utils.py b/auto_round/utils.pyindex 2f93dddcc..9c94f9216 100644--- a/auto_round/utils.py+++ b/auto_round/utils.py@@ -547,7 +547,7 @@ def detect_device_count():return 0-def detect_device(device=None):+def detect_device(device: Union[str, int, torch.device] = None) -> str:"""Detects the appropriate computation device.This function determines the device to use for computations. It can take@@ -2191,8 +2191,13 @@ def get_reciprocal(tensor):return torch.where(tensor != 0, 1 / tensor, torch.zeros_like(tensor))-def check_need_act_calibration(is_act_dynamic, act_data_type=None):- if not is_act_dynamic:+def check_need_act_calibration(+ is_act_dynamic: Union[bool, None], act_data_type: Union[str, None] = None, act_bits: int = 16+) -> bool:+ if act_bits > 8:+ return False+ # None is dynamic+ if is_act_dynamic is not None and not is_act_dynamic:return Trueif act_data_type is not None and "static" in act_data_type:return True@@ -2552,3 +2557,23 @@ def is_static_wfp8afp8(ar):if is_wfp8afp8(ar):return Truereturn False+++def get_max_vram(ratio: float = 0.9) -> dict:+ max_memory = {}+ if torch.cuda.is_available(): # NVIDIA CUDA+ num_devices = torch.cuda.device_count()+ for i in range(num_devices):+ total_mem = torch.cuda.get_device_properties(i).total_memory+ max_mem_gb = int(total_mem / 1024**3 * ratio)+ max_memory[i] = f"{max_mem_gb}GiB"+ elif torch.xpu.is_available(): # TODO need verification+ num_devices = torch.xpu.device_count()+ for i in range(num_devices):+ total_mem = torch.xpu.get_device_properties(i).total_memory+ max_mem_gb = int(total_mem / 1024**3 * ratio)+ max_memory[i] = f"{max_mem_gb}GiB"++ else:+ raise RuntimeError("No CUDA or XPU devices found.")+ return max_memorydiff --git a/test/test_cpu/test_autoround.py b/test/test_cpu/test_autoround.pyindex 43dbe2365..ccdf5b419 100644--- a/test/test_cpu/test_autoround.py+++ b/test/test_cpu/test_autoround.py@@ -636,6 +636,15 @@ class TestAutoRound(unittest.TestCase):):raise ValueError("mixed bits is not correct")+ def test_invalid_layer_config(self):+ with self.assertRaises(ValueError):+ layer_config = {"model.decoder.layers.2.self_attnx": {"bits": 2}}+ ar = AutoRound("facebook/opt-125m", scheme="W3A16", nsamples=1, iters=1, layer_config=layer_config)+ ar.quantize()+ with self.assertRaises(ValueError):+ layer_config = {"model.decoder.layers.2.self_attn": {"bit": 2}} # should be bits+ ar = AutoRound("facebook/opt-125m", scheme="W3A16", nsamples=1, iters=1, layer_config=layer_config)+if __name__ == "__main__":unittest.main()diff --git a/test/test_cpu/test_basic_usage.py b/test/test_cpu/test_basic_usage.pyindex 02db860fe..ad9e9b56c 100644--- a/test/test_cpu/test_basic_usage.py+++ b/test/test_cpu/test_basic_usage.py@@ -20,11 +20,10 @@ class TestAutoRoundCmd(unittest.TestCase):python_path = sys.executable##test llm script- # res = os.system(- # f"cd ../.. && {python_path} -m auto_round -h")- # if res > 0 or res == -1:- # assert False, "cmd line test fail, please have a check"- #+ res = os.system(f"cd ../.. && {python_path} -m auto_round -h")+ if res > 0 or res == -1:+ assert False, "cmd line test fail, please have a check"+res = os.system(f"cd ../.. && {python_path} -m auto_round --model 'facebook/opt-125m' --seqlen 32 --iter 2 --nsamples 1 --format auto_gptq,auto_round --output_dir ./saved --tasks piqa")diff --git a/test/test_cpu/test_export.py b/test/test_cpu/test_export.pyindex 68942ac77..6d1cd8b46 100644--- a/test/test_cpu/test_export.py+++ b/test/test_cpu/test_export.py@@ -25,7 +25,7 @@ class TestAutoRound(unittest.TestCase):def setUpClass(self):model_name = "facebook/opt-125m"self.save_dir = "./saved"- self.model = AutoModelForCausalLM.from_pretrained(model_name, torch_dtype="auto", trust_remote_code=True)+ self.model = AutoModelForCausalLM.from_pretrained(model_name, dtype="auto", trust_remote_code=True)self.tokenizer = AutoTokenizer.from_pretrained(model_name, trust_remote_code=True)self.llm_dataloader = LLMDataLoader()@@ -47,6 +47,7 @@ class TestAutoRound(unittest.TestCase):seqlen=2,dataset=self.llm_dataloader,)+autoround.quantize()quantized_model_path = "./saved"autoround.save_quantized(output_dir=quantized_model_path, inplace=False, format="auto_gptq")@@ -56,7 +57,7 @@ class TestAutoRound(unittest.TestCase):continuequantization_config = AutoRoundConfig()model = AutoModelForCausalLM.from_pretrained(- quantized_model_path, device_map="cpu", trust_remote_code=True, quantization_config=quantization_config+ quantized_model_path, device_map="auto", trust_remote_code=True, quantization_config=quantization_config)tokenizer = AutoTokenizer.from_pretrained(quantized_model_path)text = "There is a girl who likes adventure,"diff --git a/test/test_cpu/test_gguf_format.py b/test/test_cpu/test_gguf_format.pyindex d3f255397..61413c24b 100644--- a/test/test_cpu/test_gguf_format.py+++ b/test/test_cpu/test_gguf_format.py@@ -242,10 +242,8 @@ class TestGGUF(unittest.TestCase):"group_size": 32,"sym": False,"data_type": "int_asym_dq",- "scale_dtype": torch.float32,"super_bits": 6,"super_group_size": 8,- "in_blocks": False,},"model.embed_tokens": {"bits": 6, "group_size": 32, "super_bits": 6, "super_group_size": 8},"model.layers.12.mlp.gate_proj": {"bits": 3},diff --git a/test/test_cpu/test_scheme.py b/test/test_cpu/test_scheme.pynew file mode 100644index 000000000..e36d2aa98--- /dev/null+++ b/test/test_cpu/test_scheme.py@@ -0,0 +1,68 @@+import shutil+import sys+import unittest++from auto_round.schemes import QuantizationScheme++sys.path.insert(0, "../..")++from auto_round import AutoRound+++class TestAutoRound(unittest.TestCase):+ @classmethod+ def setUpClass(self):+ self.model_name = "facebook/opt-125m"+ self.save_folder = "./saved"++ @classmethod+ def tearDownClass(self):+ shutil.rmtree(self.save_folder, ignore_errors=True)+ shutil.rmtree("runs", ignore_errors=True)++ def test_gguf(self):+ ar = AutoRound("Qwen/Qwen3-0.6B", scheme="W2A16", nsamples=1, iters=1)+ ar.quantize_and_save(self.save_folder, format="gguf:q4_k_m")+ self.assertEqual(ar.bits, 4)+ shutil.rmtree(self.save_folder, ignore_errors=True)++ def test_w4a16(self):+ ar = AutoRound(self.model_name, scheme="W4A16", nsamples=1, iters=1)+ self.assertEqual(ar.bits, 4)+ ar.quantize()++ def test_w2a16_rtn(self):+ ar = AutoRound(self.model_name, scheme="W2A16", nsamples=1, iters=0)+ self.assertEqual(ar.bits, 2)+ ar.quantize()++ def test_mxfp4(self):+ ar = AutoRound(self.model_name, scheme="MXFP4", nsamples=1, iters=1)+ self.assertEqual(ar.bits, 4)+ self.assertEqual(ar.act_bits, 4)+ self.assertEqual(ar.data_type, "mx_fp")+ self.assertEqual(ar.act_data_type, "mx_fp_rceil")+ ar.quantize()++ def test_scheme_in_layer_config(self):+ layer_config = {+ "model.decoder.layers.2.self_attn": {"bits": 2},+ "model.decoder.layers.3.self_attn.v_proj": "W8A16",+ "model.decoder.layers.4.self_attn.k_proj": QuantizationScheme.from_dict({"group_size": 64}),+ }+ ar = AutoRound(self.model_name, scheme="W3A16", nsamples=1, iters=1, layer_config=layer_config)++ ar.quantize()+ for n, m in ar.model.named_modules():+ if n == "model.decoder.layers.2.self_attn.q_proj":+ self.assertEqual(m.bits, 2)+ if n == "model.decoder.layers.2.self_attn.k_proj":+ self.assertEqual(m.bits, 2)+ if n == "model.decoder.layers.3.self_attn.v_proj":+ self.assertEqual(m.bits, 8)+ if n == "model.decoder.layers.4.self_attn.k_proj":+ self.assertEqual(m.group_size, 64)+++if __name__ == "__main__":+ unittest.main()