PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度
support automatic mixed bits assignment (#851)
Wenhua Cheng0f64ed0cd12b
diff --git a/README.md b/README.mdindex bbebb901c..fcbce0be5 100644--- a/README.md+++ b/README.md@@ -27,6 +27,8 @@ and [fbaldassarri](https://huggingface.co/fbaldassarri). For usage instructions,## 🆕 What's New+[2025/10] AutoRound team proposed a fast algorithm to generate mixed bits/datatypes schemes in minutes. Please+refer to the documentation for accuracy [results](./docs/auto_scheme_acc.md) and [this guide](https://github.com/intel/auto-round/blob/main/docs/step_by_step.md#autoscheme) for usage instructions.[2025/09] AutoRound now includes experimental support for the mxfp4 and nvfp4 dtypes. For accuracy results, see the [documentation](./docs/mxnv_acc.md). We currently recommend exporting to the LLM-Compressor format.@@ -38,7 +40,7 @@ and [fbaldassarri](https://huggingface.co/fbaldassarri). For usage instructions,all bits other than 3 bits. Examplemodels: [Intel/Qwen3-235B-A22B-q2ks-mixed-AutoRound](https://huggingface.co/Intel/Qwen3-235B-A22B-q2ks-mixed-AutoRound)and [Intel/DeepSeek-R1-0528-q2ks-mixed-AutoRound](https://huggingface.co/Intel/DeepSeek-R1-0528-q2ks-mixed-AutoRound). **A more advanced algorithm** tailored for specific configurations may be available in- v0.7.1.+ v0.8.1.[2025/05] AutoRound has been integrated into **vLLM**. You can now run models in the AutoRound format directly withvLLM versions later than v0.85.post1.@@ -65,6 +67,9 @@ Support **AutoRound, AutoAWQ, AutoGPTQ, and GGUF** for maximum compatibility. De✅ **Affordable Quantization Cost**Quantize 7B models in about 10 minutes on a single GPU. Details are shown in [quantization costs](https://github.com/intel/auto-round/blob/main/docs/step_by_step.md#quantization-costs)+✅ **Fast mixed bits/data-types scheme generation**+Automatically configure in minutes, with about 2X-4X the model’s BF16 VRAM size as overhead.+✅ **10+ VLMs Support**Out-of-the-box quantization for 10+ vision-language models [example models](https://huggingface.co/collections/OPEA/vlms-autoround-675bc712fdd6a55ebaf11bfa), [support matrix](https://github.com/intel/auto-round/tree/main/auto_round/mllm#support-matrix)@@ -111,7 +116,7 @@ pip install auto-round-lib## Model Quantization (CPU/Intel GPU/Gaudi/CUDA)### CLI Usage-Please change to `auto-round-mllm` for visual-language models (VLMs) quantization. The full list of supported arguments is provided by calling `auto-round -h` on the terminal.+The full list of supported arguments is provided by calling `auto-round -h` on the terminal.```bashauto-round \diff --git a/auto_round/__init__.py b/auto_round/__init__.pyindex a7679a615..268065ba4 100644--- a/auto_round/__init__.py+++ b/auto_round/__init__.py@@ -15,7 +15,7 @@ from auto_round.autoround import AutoRound# support for old apifrom auto_round.autoround import AutoRoundLLM, AutoRoundMLLM, AutoRoundAdam, AutoRoundDiffusion-from auto_round.schemes import QuantizationScheme+from auto_round.schemes import QuantizationScheme, AutoSchemefrom auto_round.utils import LazyImportdiff --git a/auto_round/__main__.py b/auto_round/__main__.pyindex 242498afb..a3ab6f2e5 100644--- a/auto_round/__main__.py+++ b/auto_round/__main__.py@@ -12,19 +12,16 @@# See the License for the specific language governing permissions and# limitations under the License.import argparse-import loggingimport os-import reimport sysfrom auto_round.compressors import BaseCompressorfrom auto_round.eval.eval_cli import EvalArgumentParser, _eval_init, eval, eval_task_by_task-from auto_round.schemes import PRESET_SCHEMES+from auto_round.schemes import PRESET_SCHEMES, AutoSchemefrom auto_round.utils import (clear_memory,get_device_and_parallelism,get_model_dtype,- set_cuda_visible_devices,)RECIPES = {@@ -66,6 +63,11 @@ class BasicArgumentParser(argparse.ArgumentParser):help="The batch size for tuning/calibration.""Larger batch sizes may improve stability but require more memory.",)+ basic.add_argument("--avg_bits", default=None, type=float, help="for auto scheme, number of avg weight bits")+ basic.add_argument(+ "--options", default=None, type=str, help="for auto scheme, options for auto scheme, e.g. 'W4A16,W8A16'"+ )+basic.add_argument("--iters","--iter",@@ -138,6 +140,11 @@ class BasicArgumentParser(argparse.ArgumentParser):)tuning = self.add_argument_group("Tuning Arguments")+ tuning.add_argument(+ "--ignore_scale_zp_bits",+ action="store_true",+ help="for auto scheme whether ignore scale zp bits calculation ",+ )tuning.add_argument("--lr",default=None,@@ -176,7 +183,7 @@ class BasicArgumentParser(argparse.ArgumentParser):)tuning.add_argument("--scale_dtype",- default="fp16",+ default=None,choices=["fp16", "float16", "bf16", "bfloat16", "fp32", "float32"],help="Data type for quantization scales. ""fp16/bf16: lower memory, fp32: higher precision. "@@ -452,8 +459,6 @@ def tune(args):if "marlin" in args.format and args.asym is True:raise RuntimeError("marlin backend only supports sym quantization, please remove --asym")- # Must set this before import torch- # set_cuda_visible_devices(args.device_map)device_str, use_auto_mapping = get_device_and_parallelism(args.device_map)import torch@@ -549,6 +554,15 @@ def tune(args):extra_config.mllm_config = mllm_configextra_config.diffusion_config = diffusion_config+ layer_config = {}++ if args.avg_bits is not None:+ if args.options is None:+ raise ValueError("please set --options for auto scheme")+ scheme = AutoScheme(+ options=args.options, avg_bits=args.avg_bits, ignore_scale_zp_bits=args.ignore_scale_zp_bits+ )+autoround: BaseCompressor = AutoRound(model=model_name,scheme=scheme,@@ -565,6 +579,7 @@ def tune(args):not_use_best_mse=args.not_use_best_mse,enable_adam=args.adam,extra_config=extra_config,+ layer_config=layer_config,)model_name = args.model.rstrip("/")diff --git a/auto_round/auto_scheme/__init__.py b/auto_round/auto_scheme/__init__.pynew file mode 100644index 000000000..f4a3d2b23--- /dev/null+++ b/auto_round/auto_scheme/__init__.py@@ -0,0 +1,42 @@+# Copyright (c) 2025 Intel Corporation+#+# Licensed under the Apache License, Version 2.0 (the "License");+# you may not use this file except in compliance with the License.+# You may obtain a copy of the License at+#+# http://www.apache.org/licenses/LICENSE-2.0+#+# Unless required by applicable law or agreed to in writing, software+# distributed under the License is distributed on an "AS IS" BASIS,+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+# See the License for the specific language governing permissions and+# limitations under the License.++AUTO_SCHEME_METHODS = {}+++def register_scheme_methods(names):+ """Class decorator to register a mixed precision algorithm to the registry.++ Decorator function used before a Pattern subclass.++ Args:+ names: A string. Define the export type.++ Returns:+ cls: The class of register.+ """++ def register(alg):+ if isinstance(names, (tuple, list)):+ for name in names:+ AUTO_SCHEME_METHODS[name] = alg+ else:+ AUTO_SCHEME_METHODS[names] = alg++ return alg++ return register+++import auto_round.auto_scheme.default_algdiff --git a/auto_round/auto_scheme/default_alg.abi3.so b/auto_round/auto_scheme/default_alg.abi3.sonew file mode 100644index 000000000..9dac1fe7bBinary files /dev/null and b/auto_round/auto_scheme/default_alg.abi3.so differdiff --git a/auto_round/auto_scheme/gen_auto_scheme.py b/auto_round/auto_scheme/gen_auto_scheme.pynew file mode 100644index 000000000..7e755a30a--- /dev/null+++ b/auto_round/auto_scheme/gen_auto_scheme.py@@ -0,0 +1,149 @@+# Copyright (c) 2025 Intel Corporation+#+# Licensed under the Apache License, Version 2.0 (the "License");+# you may not use this file except in compliance with the License.+# You may obtain a copy of the License at+#+# http://www.apache.org/licenses/LICENSE-2.0+#+# Unless required by applicable law or agreed to in writing, software+# distributed under the License is distributed on an "AS IS" BASIS,+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+# See the License for the specific language governing permissions and+# limitations under the License.+import math+from dataclasses import asdict+from typing import Iterable, Union++import torch++from auto_round import AutoScheme+from auto_round.auto_scheme import AUTO_SCHEME_METHODS+from auto_round.auto_scheme.utils import compute_avg_bits_for_scheme+from auto_round.export.export_to_gguf.config import GGUF_INNER_CONFIG+from auto_round.logger import logger+from auto_round.utils import _gguf_type_fallback, get_layer_features, get_module+++class GenScheme:+ """Generate and validate quantization schemes for model layers."""++ def __init__(+ self,+ auto_scheme: AutoScheme, # TODO support shared layer+ model: torch.nn.Module,+ quant_layer_names: Iterable[str],+ fixed_layer_scheme: dict[str, dict],+ dataset: str = "pile-10k", # TODO use auto-round dataset+ device_map: Union[str, torch.device, int, dict, None] = None,+ tokenizer=None,+ enable_torch_compile=False,+ ):+ self.auto_scheme = auto_scheme+ self.model = model+ self.tokenizer = tokenizer+ self.quant_layer_names = quant_layer_names+ self.fixed_layer_scheme = fixed_layer_scheme+ self.dataset = dataset+ self.device_map = device_map if self.auto_scheme.device_map is None else self.auto_scheme.device_map+ self.enable_torch_compile = enable_torch_compile+ self._check_configs()++ def _check_configs(self) -> None:+ """Validate auto_scheme configuration and ensure avg_bits target is valid."""+ if isinstance(self.model, torch.nn.Module) and self.tokenizer is None:+ raise ValueError("tokenizer must not be None if model is nn.Module")++ if not isinstance(self.dataset, str):+ raise TypeError("`dataset` must be a string, got {type(self.dataset).__name__}.")++ min_avg_bit, max_avg_bit = self.compute_avg_bit_range()+ target = self.auto_scheme.avg_bits++ logger.info("Average bits range: [%.3f, %.3f], target = %.3f", min_avg_bit, max_avg_bit, target)+ if abs(target - min_avg_bit) < 1e-3 or abs(target - max_avg_bit) < 1e-3:+ if abs(target - min_avg_bit) < 1e-3:+ target = min_avg_bit+ else:+ target = max_avg_bit+ self.auto_scheme.avg_bits = target++ if not (min_avg_bit <= target <= max_avg_bit):+ raise ValueError(+ f"Target avg_bits={target:.3f} is outside the valid range " f"[{min_avg_bit:.3f}, {max_avg_bit:.3f}]."+ )++ def get_layer_config(self) -> dict[str, dict]:+ method_name = self.auto_scheme.method+ method_func = AUTO_SCHEME_METHODS[method_name]+ layer_config = method_func(+ self.auto_scheme,+ self.model,+ self.quant_layer_names,+ self.fixed_layer_scheme,+ self.dataset,+ self.tokenizer,+ device_map=self.device_map,+ enable_torch_compile=self.enable_torch_compile,+ )+ layer_config = self.fallback_gguf_layer_config(layer_config)+ return layer_config++ def fallback_gguf_layer_config(self, layer_config: dict[str, dict]) -> dict[str, dict]:+ """+ Apply fallback configurations for GGUF quantized layers when the current+ layer configuration is incompatible with input feature alignment.++ Args:+ layer_config (dict[str, dict]): Mapping from layer name to its quantization scheme.++ Returns:+ dict[str, dict]: Updated layer configuration with applied fallbacks if necessary.+ """+ for name, scheme in layer_config.items(): # TODO: add unit test (wenhua), the code is a little tricky+ if scheme.get("super_bits") is None:+ continue # Skip non-GGUF k-quant layers++ layer = get_module(self.model, name)+ input_features, out_features = get_layer_features(layer)+ if input_features is None:+ continue+ if input_features % 256 == 0 or isinstance(layer, torch.nn.Embedding):+ continue++ # Determine fallback quantization type+ if input_features % 256 != 0 and input_features % 32 != 0:+ new_type = "gguf:bf16"+ elif input_features % 256 != 0:+ bits = scheme["bits"]+ prefix_idx = 0 if scheme["sym"] else 1+ new_type = f"gguf:q{bits}_" + f"{prefix_idx}"+ if new_type not in GGUF_INNER_CONFIG:+ new_type = f"gguf:q{bits}_" + f"{1 - prefix_idx}"+ if new_type not in GGUF_INNER_CONFIG:+ current_type = f"gguf:q{bits}_k"+ new_type = _gguf_type_fallback(current_type)++ # Apply fallback configuration+ target_config = GGUF_INNER_CONFIG[new_type]+ for key in scheme.keys():+ if key in target_config:+ scheme[key] = target_config[key]++ logger.warning(f"Fallback applied: {name} → {new_type}")++ return layer_config++ def compute_avg_bit_range(self) -> tuple[float, float]:+ """Compute the min and max average bitwidths among candidate quantization options."""+ avg_bits = [+ compute_avg_bits_for_scheme(+ self.model,+ self.quant_layer_names,+ self.fixed_layer_scheme,+ option,+ self.auto_scheme.ignore_scale_zp_bits,+ )[0]+ for option in self.auto_scheme.options+ ]+ return min(avg_bits), max(avg_bits)diff --git a/auto_round/auto_scheme/utils.py b/auto_round/auto_scheme/utils.pynew file mode 100644index 000000000..73191c40f--- /dev/null+++ b/auto_round/auto_scheme/utils.py@@ -0,0 +1,437 @@+# Copyright (c) 2025 Intel Corporation+#+# Licensed under the Apache License, Version 2.0 (the "License");+# you may not use this file except in compliance with the License.+# You may obtain a copy of the License at+#+# http://www.apache.org/licenses/LICENSE-2.0+#+# Unless required by applicable law or agreed to in writing, software+# distributed under the License is distributed on an "AS IS" BASIS,+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+# See the License for the specific language governing permissions and+# limitations under the License.+import re+from dataclasses import asdict, fields+from typing import Iterable, Union++import torch+from accelerate import dispatch_model, infer_auto_device_map+from accelerate.utils import get_balanced_memory++from auto_round.low_cpu_mem import get_module+from auto_round.schemes import QuantizationScheme, preset_name_to_scheme+from auto_round.utils import (+ SUPPORTED_LAYER_TYPES,+ check_to_quantized,+ get_block_names,+ get_layer_features,+ is_hpex_available,+)+++def apply_quant_scheme(+ model: torch.nn.Module,+ quant_layer_names: Iterable[str],+ fixed_layer_scheme: dict[str, dict],+ scheme: Union[str, dict], # TODO add scale_dtype+) -> None:+ """Apply a quantization scheme to each quantized layer.++ Args:+ model: The model whose layers are to be updated.+ scheme: The scheme preset name or dictionary to apply.+ quant_layer_names: Iterable of layer names to quantize.+ fixed_layer_scheme: Dictionary of fixed per-layer quantization schemes.+ """+ for name in quant_layer_names:+ layer_scheme = fixed_layer_scheme.get(name, scheme)+ if isinstance(layer_scheme, str):+ layer_scheme = asdict(preset_name_to_scheme(layer_scheme))++ module = get_module(model, name)+ for key, value in layer_scheme.items():+ setattr(module, key, value)+++def remove_quant_scheme(+ model: torch.nn.Module,+) -> None:+ """Remove attributes corresponding to the applied quantization scheme.++ Args:+ model: The model whose layers are to be cleared.+ """+ scheme_keys = [f.name for f in fields(QuantizationScheme)] + ["scale_dtype"]+ for n, m in model.named_modules():+ for key in scheme_keys:+ if hasattr(m, key):+ delattr(m, key)+++def compute_avg_bits_for_scheme(+ model: torch.nn.Module,+ quant_layer_names: Iterable[str],+ fixed_layer_scheme: dict[str, dict],+ scheme: Union[str, dict, None] = None,+ ignore_scale_zp_bits: bool = False,+) -> tuple[float, float]:+ """Compute the average and total bit usage for the given quantization scheme.++ Args:+ model: The model to analyze.+ quant_layer_names: Iterable of layer names to include.+ fixed_layer_scheme: Dictionary of fixed per-layer quantization schemes.+ scheme: Optional scheme to temporarily apply before measuring.+ ignore_scale_zp_bits: If True, ignores overhead from scale and zero-points.++ Returns:+ A tuple (avg_bits, total_quantized_bits):+ avg_bits: Average bitwidth per parameter.+ total_quantized_bits: Total quantized bit count.+ """+ if scheme is not None:+ apply_quant_scheme(model, quant_layer_names, fixed_layer_scheme, scheme)++ total_params = 0+ total_quantized_bits = 0++ for name in quant_layer_names:+ module = get_module(model, name)+ # if isinstance(module,torch.nn.Embedding):+ # continue+ if not hasattr(module, "weight"):+ continue+ total_params += module.weight.numel()+ layer_bits, _ = compute_layer_bits(module, ignore_scale_zp_bits)+ total_quantized_bits += layer_bits++ avg_bits = float(total_quantized_bits) / total_params++ if scheme is not None:+ remove_quant_scheme(model)++ return avg_bits, total_quantized_bits+++def compute_avg_bits_for_model(model: torch.nn.Module, ignore_scale_zp_bits: bool = False):+ """Compute the average and total bit usage for the entire model.++ Args:+ model: The model to analyze.+ ignore_scale_zp_bits: If True, ignores overhead from scale and zero-points.+ if scheme is not None:+ apply_quant_scheme(model, quant_layer_names, fixed_layer_scheme, scheme)+ """++ total_params = 0+ total_quantized_bits = 0++ for n, module in model.named_modules():+ if not hasattr(module, "bits"):+ continue+ if not hasattr(module, "weight"):+ continue+ # if isinstance(module,torch.nn.Embedding): # Tricky setting for Embedding+ # continue+ total_params += module.weight.numel()+ layer_bits, _ = compute_layer_bits(module, ignore_scale_zp_bits)+ total_quantized_bits += layer_bits++ avg_bits = float(total_quantized_bits) / total_params++ return avg_bits, total_quantized_bits+++def compute_layer_bits(+ layer: torch.nn.Module,+ ignore_scale_zp_bits: bool = False,+) -> tuple[int, float]:+ """Compute total and average bitwidth for a single quantized layer.++ Args:+ layer: A PyTorch layer with quantization attributes.+ ignore_scale_zp_bits: Whether to ignore scale/zero-point overhead.++ Returns:+ A tuple (total_bits, avg_bits) representing bit usage.+ """+ weight = layer.weight+ n_param = weight.numel()+ weight_bits = getattr(layer, "bits", 16)+ group_size = getattr(layer, "group_size", 128)+ super_group_size = getattr(layer, "super_group_size", None)+ super_weight_bits = getattr(layer, "super_bits", None)++ # Unquantized layer or ignoring scale/zp overhead+ if weight_bits >= 16 or ignore_scale_zp_bits:+ if super_weight_bits is not None: # reset gguf 16 bits to 32 bits, TODO gguf q4_0, q4_1 have bug (wenhua)+ if weight_bits >= 16:+ return 32 * n_param, 32++ return weight_bits * n_param, min(16, weight_bits)++ in_features, out_features = get_layer_features(layer)++ # Determine number of groups based on group size+ if group_size > 0:+ n_group = out_features * (in_features + group_size - 1) // group_size+ elif group_size == 0:+ n_group = 1+ elif group_size == -1:+ n_group = out_features+ else:+ raise ValueError(f"Invalid group_size {group_size}")++ # Compute auxiliary bits (scales, zero-points, or double quantization)+ aux_total_bits = 0+ if not super_group_size:+ scale_bits = 16+ zp_bits = weight_bits+ aux_total_bits = n_group * (scale_bits + zp_bits)+ else:+ aux_total_bits += n_group * super_weight_bits * 2+ n_super_group = (n_group + super_group_size - 1) // super_group_size+ aux_total_bits += n_super_group * 32 * 2 # 32-bit scale and min_v++ total_bits = weight_bits * n_param + aux_total_bits+ avg_bits = total_bits / n_param+ return total_bits, avg_bits+++def parse_all_available_device(device_map: Union[str, torch.device, int, dict, None] = None) -> list:+ """+ Parse the device map and return a list of all available devices.++ Supported input formats:+ - None: Automatically detect all available devices+ - int: A single device index (e.g., 0)+ - str: Examples:+ "cpu"+ "cuda:0,cuda:1"+ "0,1" (numeric device indices)+ - dict: Extract all device values from the dictionary+ - torch.device: e.g. torch.device("cuda:0")++ Returns:+ list[str]: Normalized device names, e.g., ["cuda:0", "cuda:1"] or ["cpu"]+ """++ # === Step 1. Detect available device types ===+ device_types = []+ if torch.cuda.is_available():+ device_types.append("cuda")+ if hasattr(torch, "xpu") and torch.xpu.is_available():+ device_types.append("xpu")+ if hasattr(torch, "hpu") and is_hpex_available():+ device_types.append("hpu")++ # Always include CPU as a fallback+ if not device_types:+ device_types = ["cpu"]++ # === Step 2. Parse different input formats ===+ if device_map is None:+ # Automatically detect one available device+ if "cuda" in device_types:+ return ["cuda:0"]+ elif "xpu" in device_types:+ return ["xpu:0"]+ elif "hpu" in device_types:+ return ["hpu:0"]+ else:+ return ["cpu"]++ if isinstance(device_map, torch.device):+ # Handle torch.device objects+ dev_type = device_map.type+ index = device_map.index+ if dev_type == "cpu":+ return ["cpu"]+ if index is None:+ index = 0+ return [f"{dev_type}:{index}"]++ if isinstance(device_map, int):+ # Integer input → use primary available device type+ device_type = device_types[0]+ return [f"{device_type}:{device_map}"] if device_type != "cpu" else ["cpu"]++ if isinstance(device_map, str):+ # Remove whitespace+ device_map = device_map.strip()+ if device_map.lower() == "cpu":+ return ["cpu"]++ # Split by commas+ parts = [x.strip() for x in device_map.split(",") if x.strip()]+ parsed = []+ for p in parts:+ if p.isdigit():+ # Numeric → assign to first available device type+ device_type = device_types[0]+ parsed.append(f"{device_type}:{p}" if device_type != "cpu" else "cpu")+ else:+ parsed.append(p)+ return parsed++ if isinstance(device_map, dict):+ # Extract all devices recursively from dict values+ devices = set()+ for v in device_map.values():+ devices.update(parse_all_available_device(v))+ return sorted(devices)++ raise TypeError(f"Unsupported device_map type: {type(device_map)}")+++# Important Notice This dispatch does not follow dict device_map, just extract all available devices and use them+def dispatch_model_by_all_available_devices(+ model: torch.nn.Module, device_map: Union[str, int, dict, None]+) -> torch.nn.Module:+ if device_map is None:+ device_map = 0++ no_split_modules = getattr(model, "_no_split_modules", [])+ if device_map == "auto":+ max_memory = get_balanced_memory(+ model,+ max_memory=None,+ no_split_module_classes=no_split_modules,+ )+ device_map = infer_auto_device_map(model, max_memory=max_memory, no_split_module_classes=no_split_modules)+ model = dispatch_model(model, device_map=device_map)+ return model++ devices = parse_all_available_device(device_map)++ if len(devices) == 1:+ model.to(devices[0])+ return model++ max_memory = get_balanced_memory(+ model,+ max_memory=None,+ no_split_module_classes=no_split_modules,+ )++ # Filter max_memory with devices+ # assume only one GPU model+ new_max_memory = {}+ for device in devices:+ if ":" in device:+ device = int(device.split(":")[-1])+ elif device == "cpu":+ device = "cpu"+ else:+ raise ValueError(f"Unsupported device {device} in device_map: {device_map}")+ new_max_memory[device] = max_memory[device]++ device_map = infer_auto_device_map(model, max_memory=max_memory, no_split_module_classes=no_split_modules)+ model = dispatch_model(model, device_map=device_map)+ return model+++def merge_lists_unionfind(list_of_lists):+ parent = {}++ def find(x):+ while parent[x] != x:+ parent[x] = parent[parent[x]]+ x = parent[x]+ return x++ def union(x, y):+ root_x, root_y = find(x), find(y)+ if root_x != root_y:+ parent[root_y] = root_x++ # 初始化并查集+ for lst in list_of_lists:+ for item in lst:+ if item not in parent:+ parent[item] = item+ for i in range(1, len(lst)):+ union(lst[0], lst[i])++ # 收集结果+ groups = {}+ for item in parent:+ root = find(item)+ groups.setdefault(root, []).append(item)+ return list(groups.values())+++def parse_shared_layers(model: torch.nn.Module, shared_patterns: Iterable[Iterable[str]]) -> list[list[str]]:+ """+ Parse shared layer groups based on regex or substring matches.++ Args:+ model (torch.nn.Module): The model whose modules will be analyzed.+ shared_patterns (Iterable[Iterable[str]]):+ Each inner iterable defines one shared group. Each element can be:+ - a string: checked by full-name or substring match+ - a regex pattern: checked by re.fullmatch or re.search++ Returns:+ list[list[str]]: A list of matched shared layer groups.+ """+ if not shared_patterns:+ return []+ # Retrieve all high-level block names (for example, transformer blocks)+ for n, m in model.named_modules():+ m.tmp_name = n # attach global name++ block_names = get_block_names(model, quant_vision=True)+ block_names = [item for sublist in block_names for item in sublist]++ # Collect all supported layer names from the model+ supported_layer_names = [name for name, module in model.named_modules() if type(module) in SUPPORTED_LAYER_TYPES]++ # Separate groups into those already fully matched and those requiring pattern matching+ direct_match_groups = []+ fuzzy_match_groups = []+ for group in shared_patterns:+ match_status = {name: (name in supported_layer_names) for name in group}+ if all(match_status.values()):+ direct_match_groups.append(list(match_status.keys()))+ else:+ fuzzy_match_groups.append(match_status)++ matched_groups = list(direct_match_groups)++ # Search each block for modules matching remaining patterns+ for block_name in block_names:+ block_module = get_module(model, block_name)+ block_layer_local_names = [+ name for name, module in block_module.named_modules() if type(module) in SUPPORTED_LAYER_TYPES+ ]+ block_layer_names = []+ for name in block_layer_local_names:+ module = get_module(block_module, name)+ block_layer_names.append(module.tmp_name)++ for group in fuzzy_match_groups:+ matched_layers = set()+ for pattern, is_direct in group.items():+ if is_direct:+ matched_layers.add(pattern)+ continue++ for layer_name in block_layer_names:+ # Try regex match first+ try:+ if re.fullmatch(pattern, layer_name) or re.search(pattern, layer_name):+ matched_layers.add(layer_name)+ continue+ except re.error:+ pass # Not a valid regex, fallback to substring matching++ # Substring or partial match+ if pattern in layer_name:+ matched_layers.add(layer_name)++ if matched_layers:+ matched_groups.append(sorted(matched_layers))+ matched_groups = merge_lists_unionfind(matched_groups)+ return matched_groupsdiff --git a/auto_round/autoround.py b/auto_round/autoround.pyindex 68420d65c..a335b343c 100644--- a/auto_round/autoround.py+++ b/auto_round/autoround.py@@ -26,7 +26,7 @@ from auto_round.compressors import (MLLMCompressor,)from auto_round.logger import deprecated, logger-from auto_round.schemes import QuantizationScheme+from auto_round.schemes import AutoScheme, QuantizationSchemefrom auto_round.utils import is_diffusion_model, is_mllm_model@@ -64,7 +64,7 @@ class AutoRound:cls,model: Union[torch.nn.Module, str],tokenizer=None,- scheme: Union[str, dict, QuantizationScheme] = "W4A16",+ scheme: Union[str, dict, QuantizationScheme, AutoScheme] = "W4A16",layer_config: dict[str, Union[str, dict, QuantizationScheme]] = None,dataset: Union[str, list, tuple, torch.utils.data.DataLoader] = "NeelNanda/pile-10k",iters: int = 200,diff --git a/auto_round/compressors/base.py b/auto_round/compressors/base.pyindex 3a7c1f823..1616d4037 100644--- a/auto_round/compressors/base.py+++ b/auto_round/compressors/base.py@@ -26,6 +26,7 @@ from typing import Any, Callable, Unionimport accelerateimport torchfrom accelerate.big_modeling import dispatch_model, infer_auto_device_map+from accelerate.utils import get_balanced_memoryfrom torch import autocastfrom tqdm import tqdmfrom transformers import set_seed@@ -36,7 +37,7 @@ from auto_round.export.export_to_autoround import AutoRoundFormatfrom auto_round.export.export_to_gguf.config import GGUF_CONFIG, GGUF_INNER_CONFIG, ModelTypefrom auto_round.logger import loggerfrom auto_round.low_cpu_mem.utils import get_layers_before_block-from auto_round.schemes import QuantizationScheme, preset_name_to_scheme+from auto_round.schemes import AutoScheme, QuantizationScheme, get_gguf_scheme, preset_name_to_schemefrom auto_round.sign_sgd import SignSGDfrom auto_round.special_model_handler import _handle_moe_modelfrom auto_round.utils import (@@ -76,7 +77,6 @@ from auto_round.utils import (get_lm_head_name,get_max_vram,get_module,- get_quant_keys,get_shared_keys,htcore,infer_bits_by_data_type,@@ -92,6 +92,7 @@ from auto_round.utils import (mv_module_from_gpu,reset_params,set_amax_for_all_moe_layers,+ set_layer_config,set_module,to_device,to_dtype,@@ -130,7 +131,7 @@ class BaseCompressor(object):self,model: Union[torch.nn.Module, str],tokenizer=None,- scheme: Union[str, dict, QuantizationScheme] = "W4A16",+ scheme: Union[str, dict, QuantizationScheme, AutoScheme] = "W4A16",layer_config: dict[str, Union[str, dict, QuantizationScheme]] = None,dataset: Union[str, list, tuple, torch.utils.data.DataLoader] = "NeelNanda/pile-10k",iters: int = 200,@@ -200,12 +201,46 @@ class BaseCompressor(object):... "act_group_size": None,... "act_sym": None,... },+ ... "layer2": {+ ... "W8A16"+ ... }... # ...... }"""- self.scheme = None- self._parse_and_set_scheme(scheme, kwargs)+ if isinstance(scheme, AutoScheme):+ if len(scheme.options) <= 0:+ raise ValueError("options of AutoScheme must not be empty")+ options = []+ for option in scheme.options:+ new_option = self._parse_and_set_scheme(option, kwargs)+ options.append(new_option)+ scheme.options = options+ for opt in options:+ if isinstance(opt, str) and opt == "BF16":+ continue+ if isinstance(opt, QuantizationScheme):+ if opt.bits >= 16 and (opt.act_bits is None or opt.act_bits >= 16):+ continue+ self.scheme = opt # Choose the first one that not 16 bits+ break++ # apply scheme to set default bits+ self._parse_and_set_scheme(self.scheme, kwargs)++ self.is_auto_scheme = True++ else:+ self.scheme = self._parse_and_set_scheme(scheme, kwargs)+ self.is_auto_scheme = False++ scheme_keys = [f.name for f in fields(QuantizationScheme)]+ for key in scheme_keys:+ kwargs.pop(key, None)++ gguf_scheme_name = get_gguf_scheme(self.scheme)+ # GGUF uses fp32 scale dtype as default+ scale_dtype = kwargs.pop("scale_dtype", "fp32") if gguf_scheme_name else kwargs.pop("scale_dtype", "fp16")# Extra/legacy kwargs for backward compatibility# Major version releases may pack them with extra configuration optionsamp = kwargs.pop("amp", True)@@ -218,7 +253,6 @@ class BaseCompressor(object):sampler = kwargs.pop("sampler", "rand")not_use_best_mse = kwargs.pop("not_use_best_mse", False)dynamic_max_gap = kwargs.pop("dynamic_max_gap", -1)- scale_dtype = kwargs.pop("scale_dtype", "fp16")nblocks = kwargs.pop("nblocks", 1)low_cpu_mem_usage = kwargs.pop("low_cpu_mem_usage", False)to_quant_block_names: Union[str, list, None] = kwargs.pop("to_quant_block_names", None)@@ -233,13 +267,17 @@ class BaseCompressor(object):self.diffusion = kwargs.pop("diffusion") if "diffusion" in kwargs else False# Scale factor for RAM usage per parameter.self.mem_per_param_scale = kwargs.pop("mem_per_param_scale", None)- fp_layers = kwargs.pop("fp_layers", None)+ self.fp_layers = kwargs.pop("fp_layers", "")+ self.layer_config = layer_config+ self.supported_types = SUPPORTED_LAYER_TYPES+ self.inner_supported_types = INNER_SUPPORTED_LAYER_TYPES+ self.scale_dtype = convert_dtype_str2torch(scale_dtype)if kwargs:logger.warning(f"unrecognized keys {list(kwargs.keys())} were passed. Please check them.")if "CUBLAS_WORKSPACE_CONFIG" not in os.environ:os.environ["CUBLAS_WORKSPACE_CONFIG"] = ":4096:8"- # deprecated, default not to use torch.use_deterministic_algorithms+ # Deprecated, default not to use torch.use_deterministic_algorithmsif not disable_deterministic_algorithms or enable_deterministic_algorithms:if not disable_deterministic_algorithms:logger.warning(@@ -256,8 +294,8 @@ class BaseCompressor(object):if isinstance(model, str):model, tokenizer, low_cpu_mem_usage = llm_load_model(model,- device="cpu",- low_cpu_mem_mode=low_cpu_mem_usage, # always load cpu first+ device="cpu", # always load cpu first+ low_cpu_mem_mode=low_cpu_mem_usage,)elif tokenizer is None and not self.diffusion and iters > 0:raise ValueError("A tokenizer must be set for non-str model input")@@ -283,6 +321,12 @@ class BaseCompressor(object):if device_map is None:device_map = 0+ self.enable_torch_compile = enable_torch_compile+ self._adjust_torch_compile(enable_torch_compile)++ if isinstance(scheme, AutoScheme):+ self.layer_config = self._gen_auto_scheme(model, scheme, dataset, device_map)+# Set device, must place after model loadingself._set_device(device_map)@@ -296,15 +340,6 @@ class BaseCompressor(object):self.device_map = Noneself._set_device_map_in_blocks(self.device_map)- not_quantize_layer_names = get_fp_layer_names(self.model, fp_layers)- if len(not_quantize_layer_names) > 0:- logger.info(f"{not_quantize_layer_names} will not be quantized.")- if layer_config is None:- layer_config = {}- for name in not_quantize_layer_names:- layer_config[name] = {"bits": 16, "act_bits": 16, "data_type": "float", "act_data_type": "float"}- self._parse_layer_config(layer_config) # must place after model init-# Tuning hyperparametersself.seed = seedset_seed(self.seed)@@ -341,7 +376,6 @@ class BaseCompressor(object):if self.static_kv_dtype is not None:logger.warning("The static kv is experimental and currently has limited support.")- self.scale_dtype = convert_dtype_str2torch(scale_dtype)self._set_amp_dtype()self.cache_device = torch.device("cpu") if self.low_gpu_mem_usage else self.deviceif self.act_bits <= 8 and self.amp_dtype == torch.float16:@@ -353,14 +387,11 @@ class BaseCompressor(object):logger.info(f"using {self.model.dtype} for quantization tuning")# Some helpers- self.supported_types = SUPPORTED_LAYER_TYPES- self.inner_supported_types = INNER_SUPPORTED_LAYER_TYPESif "hpu" in str(self.device):self.inner_supported_types = tuple(x for x in INNER_SUPPORTED_LAYER_TYPES if x != "FP8Linear")self.batch_dim = Noneself.infer_bs_coeff = 1- self.enable_torch_compile = enable_torch_compile- self._adjust_torch_compile(enable_torch_compile)+self.block_forward = compile_func(block_forward, self.device) if self.enable_torch_compile else block_forwardself._check_configs()torch.set_printoptions(precision=3, sci_mode=True)@@ -370,7 +401,82 @@ class BaseCompressor(object):import habana_frameworks.torch.core as htcore # pylint: disable=E0401import habana_frameworks.torch.hpu as hthpu # pylint: disable=E0401]- def _set_device(self, device_map):+ def _gen_auto_scheme(+ self, model: torch.nn.Module, scheme: AutoScheme, dataset: str, device_map: Union[str, int, dict, torch.device]+ ) -> dict[str, dict]:+ if self.mllm:+ logger.info("AutoScheme is not yet supported for multimodal LLMs.")+ sys.exit(-1)++ if getattr(model, "is_fp8", False):+ logger.info("AutoScheme does not currently support FP8 models.")+ sys.exit(-1)++ all_dtypes = []+ for option in scheme.options:+ # Skip pure BF16 option+ if option == "BF16":+ continue++ # Resolve the quantization scheme or data type+ dtype = "int"+ if isinstance(option, str):+ option = preset_name_to_scheme(option)++ if isinstance(option, QuantizationScheme):+ dtype = option.data_type+ elif isinstance(option, dict):+ dtype = option.get("data_type", "int")++ all_dtypes.append(dtype)++ # Check for mixed data types+ unique_dtypes = set(all_dtypes)+ if len(unique_dtypes) > 1:+ logger.warning(+ "Models with mixed data_types "+ "cannot yet be exported to real formats except GGUF. "+ "Please save the model using the `fake` format for now."+ )++ layer_config, self.has_qlayer_outside_block = set_layer_config(+ self.model,+ self.layer_config,+ self.scheme,+ self.scale_dtype,+ self.supported_types,+ self.inner_supported_types,+ self.quant_block_list,+ self.fp_layers,+ self.quant_lm_head,+ enable_gguf_official_mixed=False,+ is_mllm=self.mllm,+ )+ quant_layer_names = layer_config.keys()+ scheme_keys = {f.name for f in fields(QuantizationScheme)}+ fixed_layer_scheme_new = {+ k: {key: v[key] for key in scheme_keys & v.keys()}+ for k, v in layer_config.items()+ if v.get("fixed_by_user", False)+ }++ # mainly using quant_layers and fixed by users+ from auto_round.auto_scheme.gen_auto_scheme import GenScheme++ gen_scheme = GenScheme(+ scheme,+ self.model,+ quant_layer_names,+ fixed_layer_scheme_new,+ dataset,+ device_map=device_map,+ tokenizer=self.tokenizer,+ enable_torch_compile=self.enable_torch_compile,+ )+ layer_config = gen_scheme.get_layer_config()+ return layer_config++ def _set_device(self, device_map: Union[str, torch.device, int, dict]) -> None:if hasattr(self, "device") and self.device is not None:returnif isinstance(device_map, (str, torch.device, int)):@@ -394,65 +500,16 @@ class BaseCompressor(object):else:raise TypeError(f"device_map should be [str, torch.device, int, dict], but got {type(device_map)}")- def _parse_layer_config(self, layer_config: dict[str, Union[str, dict, QuantizationScheme]]) -> None:- """Parse and set the layer-wise quantization configuration."""- # Some other quantization configs- self.layer_config = copy.deepcopy(layer_config) if layer_config is not None else {}- scheme_keys = {f.name for f in fields(QuantizationScheme)}-- for key, item in self.layer_config.items():- if isinstance(item, str):- config = asdict(preset_name_to_scheme(item.upper()))- elif isinstance(item, QuantizationScheme):- config = asdict(item)- elif isinstance(item, dict):- invalid_keys = set(item) - scheme_keys- if invalid_keys:- raise ValueError(- f"Invalid keys {invalid_keys} in layer_config for layer '{key}', "- f"only {scheme_keys} are supported"- )- config = dict(item)-- # Drop None values- config = {k: v for k, v in config.items() if v is not None}- self.layer_config[key] = config-- if not self.quant_lm_head or (isinstance(self.scheme, str) and self.scheme.lower().startswith("gguf")):- return- for n, _ in self.model.named_modules():- lm_head_layer_name = n-- if (- hasattr(self.model, "config")- and self.model.config.tie_word_embeddings- and hasattr(self.model, "_tied_weights_keys")- ):- tied_keys = self.model._tied_weights_keys- for item in tied_keys:- if lm_head_layer_name in item: # TODO extend to encoder-decoder layer, seq classification model- self.quant_lm_head = False- logger.warning(- "reset `quant_lm_head` to `False` as quantizing lm_head with tied weights has not been "- "supported currently"- )- break-- lm_head_layer_config = self.layer_config[lm_head_layer_name] if lm_head_layer_name in self.layer_config else {}-- for key in scheme_keys:- if key not in lm_head_layer_config:- lm_head_layer_config[key] = getattr(self, key)-- def _parse_and_set_scheme(self, scheme: Union[str, dict, QuantizationScheme], kwargs) -> None:+ def _parse_and_set_scheme(self, scheme: Union[str, dict, QuantizationScheme], kwargs) -> QuantizationScheme:"""Parse and set the quantization scheme."""+ res = ""if isinstance(scheme, QuantizationScheme):scheme = asdict(scheme)elif isinstance(scheme, dict):scheme = schemeelif isinstance(scheme, str):+ res = scheme # gguf:q4_k_s and gguf_q4_k_m has the same dict scheme, but the result is differentscheme = scheme.upper()- self.scheme = schemescheme = asdict(preset_name_to_scheme(scheme))scheme_keys = [f.name for f in fields(QuantizationScheme)]for key in scheme_keys:@@ -460,7 +517,7 @@ class BaseCompressor(object):setattr(self, key, kwargs[key])else:setattr(self, key, scheme.get(key, None))- kwargs.pop(key, None)+ # kwargs.pop(key, None)if self.act_dynamic is None:self.act_dynamic = True@@ -493,11 +550,17 @@ class BaseCompressor(object):f" match the specified 'act_bits' setting. Resetting 'act_bits' to {tmp_act_bits}.")if tmp_act_bits is not None and tmp_act_bits < 16:- for supported_dtype in SUPPORTED_DTYPES: # to easily handle dtype mx_fp4 and layer_config={xxx:{bits:8}}+ for supported_dtype in SUPPORTED_DTYPES: # To easily handle dtype mx_fp4 and layer_config={xxx:{bits:8}}if self.act_data_type.startswith(supported_dtype):- if supported_dtype + str(tmp_act_bits) == self.act_data_type: # could not replace FP8_e4m3+ if supported_dtype + str(tmp_act_bits) == self.act_data_type: # Could not replace FP8_e4m3self.act_data_type = supported_dtypebreak+ for key in scheme_keys:+ scheme[key] = getattr(self, key)+ if res and QuantizationScheme.from_dict(scheme) == preset_name_to_scheme(res):+ return res+ else:+ return QuantizationScheme.from_dict(scheme)def _adjust_torch_compile(self, enable_torch_compile: bool) -> None:"""Sets the torch compile configuration for the tuning."""@@ -741,20 +804,20 @@ class BaseCompressor(object):" We are likely to release new algorithm for certain configurations in the future.")- # Check group_size 32 for auto_round- if (- self.data_type == "int"- and hasattr(self, "formats")- and any(key in fmt for fmt in self.formats for key in ("auto_round", "auto_gptq", "auto_awq"))- ):- for n, m in self.model.named_modules():- if type(m) in self.supported_types:- if m.weight.shape[0] % 32 != 0 or m.weight.shape[1] % 32 != 0:- self.layer_config[n] = {"bits": 16}- logger.info(- f"{n} will not be quantized due to its shape not being divisible by 32,"- " resulting in an exporting issue to autogptq"- )+ # # Check group_size 32 for auto_round+ # if (+ # self.data_type == "int"+ # and hasattr(self, "formats")+ # and any(key in fmt for fmt in self.formats for key in ("auto_round", "auto_gptq", "auto_awq"))+ # ):+ # for n, m in self.model.named_modules():+ # if type(m) in self.supported_types:+ # if m.weight.shape[0] % 32 != 0 or m.weight.shape[1] % 32 != 0:+ # self.layer_config[n] = {"bits": 16}+ # logger.info(+ # f"{n} will not be quantized due to its shape not being divisible by 32,"+ # " resulting in an exporting issue to autogptq"+ # )if (self.seqlen is not None@@ -803,19 +866,26 @@ class BaseCompressor(object):formats = format.replace("q*_", f"q{self.bits}_").replace(" ", "").split(",")formats = remove_duplicates(formats) # need the keep origin order- if isinstance(self.scheme, str) and self.scheme.lower().startswith("gguf"):+ gguf_format_name = get_gguf_scheme(self.scheme)++ if gguf_format_name:for i in range(len(formats)):- if formats[i] != "fake" and formats[i] != self.scheme.lower():+ if formats[i] != "fake" and formats[i] != gguf_format_name.lower():logger.warning(- f"reset format {formats[i]} to {self.scheme.lower()} "- f"since scheme {self.scheme} can only be exported to format {self.scheme.lower()}"+ f"reset format {formats[i]} to {gguf_format_name.lower()} "+ f"since scheme {gguf_format_name} can only be exported to format {gguf_format_name.lower()}")- formats[i] = self.scheme.lower()+ formats[i] = gguf_format_name.lower()_gguf_args_check(self, formats, model_type=ModelType.TEXT)if self.mllm:_gguf_args_check(self, formats, model_type=ModelType.MMPROJ)+ for f in formats:+ if f.startswith("gguf"):+ self.scheme = f.upper()+ break+for format_ in formats:if format_ not in SUPPORTED_FORMATS:logger.error(f"Unsupported format {format_}, please choose from {SUPPORTED_FORMATS}")@@ -1299,92 +1369,6 @@ class BaseCompressor(object):for hook in hooks:hook.remove()- def _check_need_to_quantize_lm_head_embedding(self) -> bool:- """Checks if LM head and embedding layers need quantization for GGUF format.-- This function inspects the current model's formats and determines whether- it needs to apply quantization settings to the embedding and LM head layers.- The function modifies `self.layer_config` in-place and updates the model modules.-- Returns:- bool: True if the LM head needs quantization, otherwise False.-- Raises:- NotImplementedError: If multiple non-fake GGUF formats are specified.- """- gguf_scheme = False- if isinstance(self.scheme, str) and "gguf" in self.scheme.lower():- gguf_scheme = True-- if not hasattr(self, "formats") and not gguf_scheme:- return False-- has_gguf: bool = gguf_scheme or any("gguf" in fmt for fmt in self.formats)- if not has_gguf:- return False- if hasattr(self, "formats"):- formats: list[str] = [fmt for fmt in self.formats if "fake" not in fmt]- if not (len(formats) == 1 and "gguf" in formats[0]):- raise NotImplementedError("Only one GGUF format can be set at a time.")- target_format: str = formats[0]-- else:- target_format = self.scheme.lower()-- tie_word_embeddings: bool = getattr(getattr(self.model, "config", None), "tie_word_embeddings", True)- for name, module in self.model.named_modules():- if isinstance(module, torch.nn.Embedding):- key: str = "lm_head" if tie_word_embeddings else "embedding"- config: dict[str, Any] = GGUF_INNER_CONFIG[GGUF_CONFIG[target_format][key]]- self._apply_config_to_layer(name, config, True)-- if not tie_word_embeddings:- lm_head_name: str = get_lm_head_name(self.model)- config: dict[str, Any] = GGUF_CONFIG[GGUF_CONFIG[target_format]["lm_head"]]- check_fixed_by_user = (- self.layer_config[lm_head_name].get("fixed_by_user", False)- if lm_head_name in self.layer_config- else None- )- self._apply_config_to_layer(lm_head_name, config, check_fixed_by_user=check_fixed_by_user)- return True-- return False-- def _apply_config_to_layer(- self,- layer_name: str,- config: dict[str, Any],- check_fixed_by_user: bool = False,- ) -> None:- """Applies GGUF quantization configuration to a given layer.-- Args:- layer_name (str): Name of the layer to configure.- config (dict[str, Any]): GGUF layer configuration.- check_fixed_by_user (bool): If True, preserve user-defined settings.- """- act_bits: int = 16- scale_dtype: Any = self.scale_dtype- keys: list[str] = ["bits", "group_size", "super_bits", "super_group_size", "data_type", "sym"]-- self.layer_config[layer_name] = self.layer_config.get(layer_name, {})-- for key in keys:- if (- key in self.layer_config[layer_name]- and check_fixed_by_user- # and self.layer_config[layer_name].get("fixed_by_user", False)- ):- continue- self.layer_config[layer_name][key] = config.get(key)- setattr(get_module(self.model, layer_name), key, config.get(key))-- self.layer_config[layer_name]["act_bits"] = act_bits- self.layer_config[layer_name]["scale_dtype"] = scale_dtype- setattr(get_module(self.model, layer_name), "act_bits", act_bits)- setattr(get_module(self.model, layer_name), "scale_dtype", scale_dtype)-def _quantize_layer_via_rtn(self, name: str) -> None:"""Quantizes a layer using RTN (Round-To-Nearest) if available.@@ -1697,32 +1681,36 @@ class BaseCompressor(object):Returns:The quantized model and layer configurations."""- for n, m in self.model.named_modules():+ for n, m in self.model.named_modules(): # TODO check if could removedm.tmp_name = nself._check_compatibility()formats = self.formats if hasattr(self, "formats") else None# It is best to modify the model structure in the quantize function and check the format,# because it may cause the gguf format to not be exported normally.self.model = _handle_moe_model(self.model, formats=formats)- self.has_qlayer_outside_block = self._set_layerwise_config(self.layer_config)++ # TODO check scale_dtype+ if not self.is_auto_scheme:+ enable_gguf_official_mixed = True+ else:+ enable_gguf_official_mixed = False+ self.layer_config, self.has_qlayer_outside_block = set_layer_config(+ self.model,+ self.layer_config,+ self.scheme,+ self.scale_dtype,+ self.supported_types,+ self.inner_supported_types,+ self.quant_block_list,+ self.fp_layers,+ self.quant_lm_head,+ enable_gguf_official_mixed=enable_gguf_official_mixed,+ is_mllm=self.mllm,+ )+if not hasattr(self, "formats"):logger.warning("this API is deprecated, please use `quantize_and_save` instead")else:- only_gguf = True- for format_ in self.formats:- if not ("gguf" in format_ or "fake" in format_):- only_gguf = False- break- if len(self.formats) == 1 and self.formats[0] == "fake":- only_gguf = False- if only_gguf:- self.layer_config, gguf_format_config = get_layer_config_by_gguf_format(- self.layer_config, self.formats, self.model, model_type=ModelType.TEXT- )- if self.mllm:- self.layer_config, gguf_format_config = get_layer_config_by_gguf_format(- self.layer_config, self.formats, self.model, model_type=ModelType.MMPROJ- )# Determine if immediate packing is requiredformats = self.formatsif (@@ -1828,7 +1816,7 @@ class BaseCompressor(object):cost_time = end_time - self.start_timelogger.info(f"quantization tuning time {cost_time}")- ## dump a summary+ # Dump a summaryquantized_layers = []unquantized_layers = []for n, m in self.model.named_modules():@@ -1923,141 +1911,6 @@ class BaseCompressor(object):del layer_inputclear_memory(q_layer_input)- def _set_layerwise_config(self, layer_config: dict) -> bool:- """- Sets the layer-wise configuration based on the provided `layer_config`.- By default, only quantize layers in blocks.-- Args:- layer_config (dict): The configuration dictionary for each layer containing various configuration options.-- Returns:- bool: Returns True if there are quantized layers outside the blocks (e.g., lm-head),- otherwise returns False.- """- # Get the names of layers in quantization blocks- supported_types = self.supported_types- layers_in_blocks = get_layer_names_in_block(- self.model, supported_types, self.quant_block_list, self.inner_supported_types- )- ##process regex in layer_config- all_supported_layer_names = []- # List of configuration keys- keys = get_quant_keys()-- for n, m in self.model.named_modules():- # Delete previous configuration to avoid conflicts with prior tuning- for key in keys:- if hasattr(m, key):- delattr(m, key)-- if not isinstance(m, supported_types) and m.__class__.__name__ not in self.inner_supported_types:- continue- all_supported_layer_names.append(n)-- names_in_layer_config = list(layer_config.keys())- for name in names_in_layer_config:- if name in all_supported_layer_names:- continue- matched_names = []- for layer_name in all_supported_layer_names:- if re.search(re.compile(name), layer_name) is not None:- matched_names.append(layer_name)- if len(matched_names) > 0:- val = layer_config[name]- layer_config.pop(name)- for match_name in matched_names:- layer_config[match_name] = val- else:- tmp_m = get_module(self.model, name)- if not isinstance(tmp_m, torch.nn.Embedding): # TODO not good code style- raise ValueError(f"key {name} in layer_config is invalid, please have a double check")-- has_qlayer_outside_block = False # Flag to track if there are quantized layers outside blocks (e.g., lm-head)-- # Iterate through all modules in the model- is_gguf = hasattr(self, "formats") and any("gguf" in format_ for format_ in self.formats)- for n, m in self.model.named_modules():- # Skip unsupported types- if type(m) not in supported_types and m.__class__.__name__ not in self.inner_supported_types:- if n in self.layer_config:- if not isinstance(m, torch.nn.Embedding):- logger.warning(f"{n} is not supported, layer_config {n}: {layer_config[n]} will be ignored.")- self.layer_config.pop(n)- continue- if not is_gguf:- if not check_to_quantized(layer_config[n]):- self.layer_config.pop(n)- continue- else:- continue-- # If the layer is not in the config and is part of a quantization block, use default configuration- if n not in layer_config.keys() and n in layers_in_blocks:- layer_config[n] = {}- for key in keys:- layer_config[n][key] = getattr(self, key)-- # If the layer is partially configured, fill in missing values- elif n in layer_config.keys():- if "data_type" in layer_config[n] and "bits" not in layer_config[n]:- tmp_bits = infer_bits_by_data_type(layer_config[n]["data_type"])- if tmp_bits is not None and tmp_bits != self.bits:- logger.warning(- f"'data_type' do not match the specified 'bits' setting for {n}."- f" Resetting 'bits' to {tmp_bits}."- )- layer_config[n]["bits"] = tmp_bits- if "act_data_type" in layer_config[n] and "act_bits" not in layer_config[n]:- tmp_bits = infer_bits_by_data_type(layer_config[n]["act_data_type"])- if tmp_bits is not None and tmp_bits != self.act_bits:- logger.warning(- f"'act_data_type' do not match the specified 'act_bits' setting for {n}."- f" Resetting 'act_bits' to {tmp_bits}."- )- layer_config[n]["act_bits"] = tmp_bits-- for key in keys:- if key not in layer_config[n].keys():- layer_config[n][key] = getattr(self, key)- layer_config[n]["fixed_by_user"] = True-- # If the layer is not in the config and not part of a quantization block,- # use default configuration and set specific values- else:- layer_config[n] = {}- for key in keys:- layer_config[n][key] = getattr(self, key)- layer_config[n]["bits"] = 16- layer_config[n]["act_bits"] = 16-- if n in layers_in_blocks:- layer_config[n]["in_blocks"] = True- else:- layer_config[n]["in_blocks"] = False-- # If the layer is outside a block and requires quantization, mark it as a quantized layer outside the block- if (- n not in layers_in_blocks- and check_to_quantized(layer_config[n])- and not isinstance(m, torch.nn.Embedding)- ):- has_qlayer_outside_block = True-- in_features, out_features = get_layer_features(m)- if in_features <= layer_config[n]["group_size"]:- layer_config[n]["group_size"] = -1-- # Apply the configuration to the corresponding layer in the model- for key in keys:- setattr(m, key, layer_config[n][key])- need_to_quantize_lm_head = self._check_need_to_quantize_lm_head_embedding()- if need_to_quantize_lm_head:- has_qlayer_outside_block = True-- # Return whether there are quantized layers outside the blocks- return has_qlayer_outside_block-@torch.no_grad()def _get_block_outputs(self,@@ -2249,8 +2102,12 @@ class BaseCompressor(object):if str(self.model.device) == "cpu" and (self.device.startswith("xpu") or self.device.startswith("cuda")):- max_memory = get_max_vram() # TODO model is not evenly splitno_split_modules = getattr(self.model, "_no_split_modules", [])+ max_memory = get_balanced_memory(+ self.model,+ max_memory=None,+ no_split_module_classes=no_split_modules,+ )device_map = infer_auto_device_map(self.model, max_memory=max_memory, no_split_module_classes=no_split_modules)diff --git a/auto_round/data_type/gguf.py b/auto_round/data_type/gguf.pyindex 6456b817f..6aa19a3d5 100644--- a/auto_round/data_type/gguf.py+++ b/auto_round/data_type/gguf.py@@ -337,7 +337,7 @@ def quant_tensor_gguf_asym_dq(if bits == 2:quant_weights = torch.abs(tensor)elif bits == 4 or bits == 5:- sigma2 = torch.sum(torch.pow(tensor, 2), dim=-1, keepdim=True) / 32 ##Note 32 is different from QK_K+ sigma2 = torch.sum(torch.pow(tensor, 2), dim=-1, keepdim=True) / 32 # Note 32 is different from QK_Kav_x = torch.sqrt(sigma2)quant_weights = torch.abs(tensor) + av_xparams = search_kwargs[bits]diff --git a/auto_round/data_type/register.py b/auto_round/data_type/register.pyindex 12c4406a4..fca259ed6 100644--- a/auto_round/data_type/register.py+++ b/auto_round/data_type/register.py@@ -22,8 +22,7 @@ def register_dtype(names):Decorator function used before a Pattern subclass.Args:- cls (class): The subclass of register.- name: A string. Define the export type.+ names: A string. Define the export type.Returns:cls: The class of register.diff --git a/auto_round/export/export_to_autoround/export_to_nvfp_mxfp.py b/auto_round/export/export_to_autoround/export_to_nvfp_mxfp.pyindex c4a02f673..eaf3ad9ae 100644--- a/auto_round/export/export_to_autoround/export_to_nvfp_mxfp.py+++ b/auto_round/export/export_to_autoround/export_to_nvfp_mxfp.py@@ -174,7 +174,7 @@ def save_quantized_as_fp(output_dir, inplace=True, **kwargs):for n, m in model.named_modules():if type(m) in SUPPORTED_LAYER_TYPES:layer = m- if layer.act_bits < 8 and not getattr(layer, "input_global_scale", None):+ if hasattr(layer, "act_bits") and layer.act_bits < 8 and not getattr(layer, "input_global_scale", None):assert hasattr(layer, "act_max")from auto_round.data_type.nvfp import calculate_gparam@@ -198,7 +198,7 @@ def save_quantized_as_fp(output_dir, inplace=True, **kwargs):for layer_name in layer_config:if (not layer_config[layer_name]["in_blocks"] and layer_config[layer_name]["bits"] <= 8- ): ##lm head ##TODO fix act and so on+ ): ##lm head # TODO fix act and so onextra_config[layer_name] = {}extra_config[layer_name]["bits"] = layer_config[layer_name]["bits"]extra_config[layer_name]["data_type"] = layer_config[layer_name]["data_type"]diff --git a/auto_round/export/export_to_llmcompressor/export_to_fp.py b/auto_round/export/export_to_llmcompressor/export_to_fp.pyindex 1a56ebf12..633593075 100644--- a/auto_round/export/export_to_llmcompressor/export_to_fp.py+++ b/auto_round/export/export_to_llmcompressor/export_to_fp.py@@ -169,7 +169,7 @@ def save_quantized_as_fp(output_dir, inplace=True, **kwargs):for n, m in model.named_modules():if type(m) in SUPPORTED_LAYER_TYPES:layer = m- if layer.act_bits < 8 and not getattr(layer, "input_global_scale", None):+ if hasattr(layer, "act_bits") and layer.act_bits < 8 and not getattr(layer, "input_global_scale", None):assert hasattr(layer, "act_max")from auto_round.data_type.nvfp import calculate_gparamdiff --git a/auto_round/schemes.py b/auto_round/schemes.pyindex a5c5975c9..81b1cc1f1 100644--- a/auto_round/schemes.py+++ b/auto_round/schemes.py@@ -14,9 +14,11 @@import copyfrom copy import deepcopyfrom dataclasses import dataclass, fields-from typing import Generator, List, Optional+from typing import Iterable, Optional, Union-__all__ = ["QuantizationScheme", "preset_name_to_scheme"]+import torch++__all__ = ["QuantizationScheme", "get_gguf_scheme", "preset_name_to_scheme", "AutoScheme"]@dataclass@@ -38,7 +40,7 @@ class QuantizationScheme:return cls(**config)@classmethod- def get_attributes(cls: "QuantizationScheme") -> List[str]:+ def get_attributes(cls: "QuantizationScheme") -> list[str]:return [field.name for field in fields(cls)]def __getitem__(self, key: str):@@ -72,7 +74,15 @@ class QuantizationScheme:def __eq__(self, other: "QuantizationScheme") -> bool:if not isinstance(other, QuantizationScheme):return False+ skip_act_check = False+ self_act_bits = 16 if self.act_bits is None else self.act_bits+ other_act_bits = 16 if other.act_bits is None else other.act_bits+ if self_act_bits == other_act_bits and other_act_bits >= 16:+ skip_act_check = True+for field in self.get_attributes():+ if skip_act_check and field.startswith("act_"):+ continueif getattr(self, field) != getattr(other, field):return Falsereturn True@@ -114,6 +124,27 @@ W2A16 = QuantizationScheme.from_dict(})+W2A16G64 = QuantizationScheme.from_dict(+ {+ "bits": 2,+ "sym": True,+ "group_size": 64,+ "data_type": "int",+ "act_bits": 16,+ }+)+++W2A16G32 = QuantizationScheme.from_dict(+ {+ "bits": 2,+ "sym": True,+ "group_size": 32,+ "data_type": "int",+ "act_bits": 16,+ }+)+W3A16 = QuantizationScheme.from_dict({"bits": 3,@@ -180,6 +211,7 @@ FPW8A16 = QuantizationScheme.from_dict(})+# FP8 = asdict(QuantArgs.from_dict({# "bits": 8,# "group_size": 128,@@ -201,6 +233,17 @@ FP8_STATIC = QuantizationScheme.from_dict(})+# For AutoScheme 16 bits options+BF16 = QuantizationScheme.from_dict(+ {+ "bits": 16,+ "group_size": 128,+ "data_type": "fp",+ "act_bits": 16,+ "act_data_type": "fp",+ }+)+PRESET_SCHEMES = {"W4A16": W4A16,"W2A16": W2A16,@@ -210,7 +253,10 @@ PRESET_SCHEMES = {"MXFP8": MXFP8,"NVFP4": NVFP4,"FPW8A16": FPW8A16,+ "W2A16G64": W2A16G64,+ "W2A16G32": W2A16G32,"FP8_STATIC": FP8_STATIC,+ "BF16": BF16,}from auto_round.export.export_to_gguf.config import GGUF_CONFIG@@ -220,3 +266,40 @@ for key, val in GGUF_CONFIG.items():value.pop("embedding", None)value.pop("lm_head", None)PRESET_SCHEMES[key.upper()] = QuantizationScheme.from_dict(value)+++def get_gguf_scheme(scheme: Union[str, QuantizationScheme]) -> str:+ if isinstance(scheme, str) and scheme.upper().startswith("GGUF"):+ return scheme+ if isinstance(scheme, str):+ return ""+ for key, val in PRESET_SCHEMES.items():+ # For q40 or q4_1 we only support it with str scheme, otherwise it will be matched incorrectly with W4G32+ if not key.upper().startswith("GGUF") or ("0" in key or "1" in key):+ continue+ equal = True+ for scheme_key in val.keys():+ if val[scheme_key] is not None and val[scheme_key] != scheme.get(scheme_key, None):+ equal = False+ break+ if equal:+ return key+ return ""+++@dataclass+class AutoScheme:+ avg_bits: float+ options: Union[str, list[Union[QuantizationScheme, str]], tuple[Union[QuantizationScheme, str], ...]]+ shared_layers: Optional[Iterable[Iterable[str]]] = None+ method: str = "default"+ ignore_scale_zp_bits: bool = False+ nsamples: Optional[int] = None+ seqlen: Optional[int] = None+ dataset: Optional[str] = None # Import Notice no comma for each item+ device_map: Optional[Union[str, torch.device, int, dict]] = None++ def __post_init__(self):+ if isinstance(self.options, str):+ options = self.options.upper().replace(" ", "")+ self.options = options.split(",")diff --git a/auto_round/utils.py b/auto_round/utils.pyindex 0b819e381..8c8c9acc5 100644--- a/auto_round/utils.py+++ b/auto_round/utils.py@@ -21,6 +21,7 @@ import osimport reimport sysfrom collections import UserDict+from dataclasses import asdict, fieldsfrom enum import Enumfrom functools import lru_cachefrom pathlib import Path@@ -29,12 +30,13 @@ from typing import Any, Callable, Tuple, Unionimport cpuinfoimport torchimport transformers+from accelerate.utils import get_balanced_memoryfrom packaging import versionfrom torch.amp import autocastfrom auto_round.export.export_to_gguf.config import GGML_QUANT_SIZES, GGUF_CONFIG, GGUF_INNER_CONFIG, QK_K, ModelTypefrom auto_round.logger import logger-from auto_round.schemes import QuantizationScheme+from auto_round.schemes import QuantizationScheme, get_gguf_scheme, preset_name_to_schemeSHARED_CACHE_KEYS = ("position_ids", "cache_position", "position_embeddings")@@ -782,8 +784,11 @@ def check_memory_availability(device, inputs, weight, org_seqlen, org_bs):def get_layer_names_in_block(- model, supported_types=(torch.nn.Linear, transformers.pytorch_utils.Conv1D), quant_block_list=None, class_names=None-):+ model: torch.nn.Module,+ supported_types=(torch.nn.Linear, transformers.pytorch_utils.Conv1D),+ quant_block_list: list = None,+ class_names: tuple = None,+) -> list[str]:"""Retrieves the names of layers within each block of the model.Returns:@@ -807,7 +812,6 @@ def get_layer_names_in_block(if hasattr(m, "bk_tmp_name"):layers_in_block.append(m.bk_tmp_name)delattr(m, "bk_tmp_name")-return layers_in_block@@ -992,7 +996,7 @@ def compile_func_on_cuda_or_cpu(func):def compile_func(- fun: Union[torch.nn.Module, Callable], device: Union[torch.nn.Module, Callable]+ fun: Union[torch.nn.Module, Callable], device: Union[str, torch.device, int]) -> Union[torch.nn.Module, Callable]:"""Compile function on the specified device."""if "hpu" in str(device):@@ -1064,7 +1068,7 @@ def can_pack_with_numba(): # pragma: no coverreturn True-def get_fp_layer_names(model, fp_layers):+def get_fp_layer_names(model: torch.nn.Module, fp_layers: str):"""Identifies and returns layers in the model to exclude from quantization.This function processes a comma-separated list of fully precision (FP) layers,@@ -1875,7 +1879,8 @@ def _search_gguf_type(gguf_type):return None-def _gguf_type_fallback(gguf_type):+def _gguf_type_fallback(gguf_type: str) -> str:+ gguf_type = gguf_type.lower()if gguf_type in ("gguf:q2_k", "gguf:q3_k", "gguf:q4_k"):gguf_type = "gguf:q5_0"elif gguf_type == "gguf:q5_k":@@ -1886,9 +1891,9 @@ def _gguf_type_fallback(gguf_type):##https://github.com/ggml-org/llama.cpp/blob/9e31bec4fd53634c9e5b04650488a09a055f5dab/src/llama-quant.cpp#L129-def get_layer_config_by_gguf_format(layer_config, gguf_format, model, model_type=ModelType.TEXT):- # TODO: support for other format later- target_gguf_format = next((fmt for fmt in gguf_format if fmt != "fake"), None)+def get_layer_config_by_gguf_format(layer_config, target_gguf_format: str, model, model_type=ModelType.TEXT):+ # # TODO: support for other format later+ # target_gguf_format = next((fmt for fmt in gguf_format if fmt != "fake"), None)import gguf # pylint: disable=E0401@@ -1982,6 +1987,30 @@ def get_layer_config_by_gguf_format(layer_config, gguf_format, model, model_type)new_type = new_type[:bits_index] + target_bits + new_type[bits_index + 1 :]else:+ config_tmp = config.copy()+ scheme_keys = [f.name for f in fields(QuantizationScheme)]+ for key in config.keys():+ if key not in scheme_keys:+ config_tmp.pop(key, None)+ matched_scheme = get_gguf_scheme(QuantizationScheme.from_dict(config_tmp)) # check matched+ if not matched_scheme:+ if config.get("super_group_size", None) is not None:+ new_type = new_type[:bits_index] + str(config["bits"]) + "_k"+ if config.get("super_group_size", None) is None or new_type not in GGUF_INNER_CONFIG:+ prefix_idx = 0 if config.get("sym", True) else 1+ new_type = new_type[:bits_index] + str(config["bits"]) + f"_{prefix_idx}"+ if new_type not in GGUF_INNER_CONFIG:+ new_type = new_type[:bits_index] + str(config["bits"]) + f"_{1-prefix_idx}"+ if new_type not in GGUF_INNER_CONFIG:+ raise ValueError(+ f"the setting in layer_config {layer_name} "+ f"could not match any supported gguf format, please have a check."+ )+ else:+ logger.warning_once(+ f"the setting in layer_config {layer_name} "+ f"could not match any supported gguf format, reset to {new_type}"+ )new_type = new_type[:bits_index] + str(config["bits"]) + new_type[bits_index + 1 :]new_type = _search_gguf_type(new_type)if new_type is None:@@ -2233,9 +2262,9 @@ def get_reciprocal(tensor):def check_need_act_calibration(- is_act_dynamic: Union[bool, None], act_data_type: Union[str, None] = None, act_bits: int = 16+ is_act_dynamic: Union[bool, None], act_data_type: Union[str, None] = None, act_bits: Union[int, None] = 16) -> bool:- if act_bits > 8:+ if act_bits is None or act_bits > 8:return False# None is dynamicif is_act_dynamic is not None and not is_act_dynamic:@@ -2325,8 +2354,8 @@ def convert_fp8_layer_to_linear(layer, dtype=torch.bfloat16):new_layer = torch.nn.Linear(layer.in_features, layer.out_features, bias=layer.bias is not None, dtype=dtype)if layer.bias is not None:new_layer.bias.data.copy_(layer.bias.data.to(dtype=dtype))-- keys = get_quant_keys() + ["tmp_name"]+ scheme_keys = (f.name for f in fields(QuantizationScheme))+ keys = tuple(scheme_keys) + ("tmp_name", "scale_dtype")for key in keys:setattr(new_layer, key, getattr(layer, key, None))@@ -2355,24 +2384,6 @@ def convert_fp8_model_to_16b_model(model, dtype=torch.bfloat16):return model-def get_quant_keys():- keys = [- "bits",- "group_size",- "sym",- "data_type",- "scale_dtype",- "act_bits",- "act_group_size",- "act_sym",- "act_dynamic",- "act_data_type",- "super_bits",- "super_group_size",- ]- return keys--def out_of_vram(error_msg):error_msg = str(error_msg)# CUDA@@ -2805,6 +2816,199 @@ def is_mllm_model(model_or_path: Union[str, torch.nn.Module]):return False+def set_layer_config(+ model: torch.nn.Module,+ layer_config: dict[str, Union[str, dict, "QuantizationScheme"]],+ default_scheme: Union[str, "QuantizationScheme"],+ default_scale_dtype: torch.dtype | str,+ supported_types: tuple,+ inner_supported_types: tuple,+ quant_block_list=None,+ fp_layers: str = "",+ quant_lm_head: bool = False,+ enable_gguf_official_mixed: bool = True,+ is_mllm: bool = False,+) -> tuple[dict, bool]:+ """+ Normalize, validate, and expand layer-specific quantization configs.+ Returns (final_layer_config, has_quant_layer_outside_block)+ """++ from auto_round.schemes import get_gguf_scheme++ # ---- helpers -------------------------------------------------+ def dispatch_layer_config(layer_config: dict[str, dict]) -> None:+ """Assign scheme values as attributes to matched modules."""+ for layer_name, scheme in layer_config.items():+ module = get_module(model, layer_name)+ for attr, value in scheme.items():+ setattr(module, attr, value)++ def normalize_item(item: Union[str, dict, "QuantizationScheme"], layer_name: str) -> dict:+ """Convert config entry into dict and validate keys."""+ if isinstance(item, str):+ config = asdict(preset_name_to_scheme(item.upper()))+ elif isinstance(item, QuantizationScheme):+ config = asdict(item)+ elif isinstance(item, dict):+ invalid = set(item) - set(scheme_keys + ("fixed_by_user", "scale_dtype"))+ if invalid:+ raise ValueError(+ f"Invalid keys {invalid} in layer_config for '{layer_name}'. " f"Allowed keys: {scheme_keys}"+ )+ config = dict(item)+ else:+ raise TypeError(+ f"Unsupported type for layer_config[{layer_name}]: {type(item)}. "+ f"Expected str, dict, or QuantizationScheme."+ )+ # Clean up+ config = {k: v for k, v in config.items() if v is not None}+ config["fixed_by_user"] = True+ return config++ # ---- main logic ----------------------------------------------+ scheme_keys = tuple(f.name for f in fields(QuantizationScheme)) + ("scale_dtype",)+ layer_config = copy.deepcopy(layer_config) or {}++ # 1. fp_layers -> force 16+ for name in get_fp_layer_names(model, fp_layers):+ layer_config[name] = {+ "bits": 16,+ "act_bits": 16,+ "data_type": "float",+ "act_data_type": "float",+ "fixed_by_user": True,+ }++ # 2. normalize+ layer_config = {k: normalize_item(v, k) for k, v in layer_config.items()}++ # 3. infer missing bits+ for cfg in layer_config.values():+ if "data_type" in cfg and "bits" not in cfg:+ if (b := infer_bits_by_data_type(cfg["data_type"])) is not None:+ cfg["bits"] = b+ if "act_data_type" in cfg and "act_bits" not in cfg:+ if (b := infer_bits_by_data_type(cfg["act_data_type"])) is not None:+ cfg["act_bits"] = b++ # 4. fill defaults+ if isinstance(default_scheme, str):+ default_dict = asdict(preset_name_to_scheme(default_scheme.upper()))+ else:+ default_dict = asdict(default_scheme)+ default_dict["scale_dtype"] = default_scale_dtype+ for cfg in layer_config.values():+ for key in scheme_keys:+ cfg.setdefault(key, copy.deepcopy(default_dict.get(key)))++ # 5. collect supported modules+ gguf_name = get_gguf_scheme(default_scheme)+ if gguf_name and torch.nn.Embedding not in supported_types:+ supported_types = (*supported_types, torch.nn.Embedding)++ all_supported_layer_names, embedding_layer_names = [], []+ all_module_names = []+ for n, m in model.named_modules():+ all_module_names.append(n)+ # cleanup stale attributes+ for key in scheme_keys:+ if hasattr(m, key):+ delattr(m, key)+ if type(m) not in supported_types and m.__class__.__name__ not in inner_supported_types:+ continue+ all_supported_layer_names.append(n)+ if isinstance(m, torch.nn.Embedding):+ embedding_layer_names.append(n)++ # 6. expand regex configs+ for name in list(layer_config.keys()):+ if name in all_supported_layer_names:+ continue+ if name in all_module_names:+ m = get_module(model, name)+ if len(list(m.children())) == 0 and type(m) not in supported_types:+ logger.warning(f"{name} is not supported in current scheme, ignoring its setting in `layer_config`")+ continue++ regex = re.compile(name)+ matched = [ln for ln in all_supported_layer_names if regex.search(ln)]+ if not matched:+ raise ValueError(f"Invalid '{name}' in layer_config, no match found.")+ val = layer_config.pop(name)+ for match in matched:+ layer_config[match] = val++ # 7. lm_head+ lm_head_name = get_lm_head_name(model)+ tie_word_embeddings = False+ if hasattr(model, "config") and hasattr(model.config, "tie_word_embeddings"):+ tie_word_embeddings = model.config.tie_word_embeddings++ if quant_lm_head and tie_word_embeddings:+ quant_lm_head = False+ logger.warning(+ "reset `quant_lm_head` to false as quantizing " "lm_head with tied weights has not been supported currently"+ )++ if lm_head_name not in layer_config and quant_lm_head:+ layer_config[lm_head_name] = copy.deepcopy(default_dict)++ # 8. enforce shape divisibility for int weight-only+ if default_dict["data_type"] == "int" and default_dict["act_bits"] >= 16 and not gguf_name:+ for n, m in model.named_modules():+ if type(m) in supported_types or m.__class__.__name__ in inner_supported_types:+ if m.weight.shape[0] % 32 or m.weight.shape[1] % 32:+ layer_config.setdefault(n, copy.deepcopy(default_dict))+ layer_config[n].update({"bits": 16, "data_type": "fp", "fixed_by_user": True})+ logger.warning_once(f"{n} skipped quantization (shape not divisible by 32).")++ # 9. block layers: mark as in_blocks=True+ for name in get_layer_names_in_block(model, supported_types, quant_block_list, inner_supported_types):+ if name not in layer_config:+ layer_config[name] = copy.deepcopy(default_dict)+ layer_config[name]["fixed_by_user"] = False+ layer_config[name]["in_blocks"] = True++ # ---- restore: ensure missing in_blocks are set to False and compute flag ----+ has_qlayer_outside_block = False+ for cfg in layer_config.values():+ if "in_blocks" not in cfg:+ cfg["in_blocks"] = False+ # mark layer outside block+ if not cfg["in_blocks"] and check_to_quantized(cfg):+ has_qlayer_outside_block = True++ # 10. GGUF handling+ if not gguf_name:+ dispatch_layer_config(layer_config)+ return layer_config, has_qlayer_outside_block++ # embed + lm_head defaults for gguf+ if lm_head_name not in layer_config and not tie_word_embeddings:+ cfg = GGUF_INNER_CONFIG[GGUF_CONFIG[gguf_name.lower()]["lm_head"]]+ cfg = {**cfg, "fixed_by_user": False, "scale_dtype": default_scale_dtype}+ layer_config[lm_head_name] = cfg+ has_qlayer_outside_block = True+ for emd_name in embedding_layer_names:+ if emd_name in layer_config:+ continue+ if not tie_word_embeddings:+ cfg = GGUF_INNER_CONFIG[GGUF_CONFIG[gguf_name.lower()]["embedding"]]+ else:+ cfg = GGUF_INNER_CONFIG[GGUF_CONFIG[gguf_name.lower()]["lm_head"]]+ cfg = {**cfg, "fixed_by_user": False, "scale_dtype": default_scale_dtype}+ layer_config[emd_name] = cfg++ if enable_gguf_official_mixed:+ model_type = ModelType.MMPROJ if is_mllm else ModelType.TEXT+ layer_config, _ = get_layer_config_by_gguf_format(layer_config, gguf_name.lower(), model, model_type)++ dispatch_layer_config(layer_config)+ return layer_config, has_qlayer_outside_block++def check_diffusers_installed(): # pragma: no covertry:import diffusers # noqa: F401@@ -2815,7 +3019,7 @@ def check_diffusers_installed(): # pragma: no coverexit(-1)-def is_diffusion_model(model_or_path: Union[str, object]):+def is_diffusion_model(model_or_path: Union[str, object]) -> bool:if isinstance(model_or_path, str):index_file = Noneif not os.path.isdir(model_or_path):diff --git a/auto_round/wrapper.py b/auto_round/wrapper.pyindex 98fa80289..a9c0f5cb2 100644--- a/auto_round/wrapper.py+++ b/auto_round/wrapper.py@@ -158,7 +158,7 @@ class WrapperLinear(torch.nn.Module):self.act_quant_func = compile_func(self.act_quant_func, self.device)self._init_params("act_max_scale", p_dtype, (1), 1.0, not orig_layer.act_dynamic)- ## bias tuning+ # Bias tuningif self.enable_norm_bias_tuning:self._init_params("bias_v", p_dtype, self.orig_layer.bias.shape, 0, True)from auto_round.data_type.int import quant_tensor_asym_wo_rounddiff --git a/docs/alg_202508.md b/docs/alg_202508.mdindex 3fb74ace8..086cd5cf6 100644--- a/docs/alg_202508.md+++ b/docs/alg_202508.md@@ -3,12 +3,12 @@ We use **lm-eval** for evaluation. For LLaMA, we enabled `add_bos_token` andin [modeling_llama.py](https://github.com/huggingface/transformers/blob/main/src/transformers/models/llama/modeling_llama.py#L52C1-L52C40)to stabilize accuracy during evaluation. All other settings follow the default configurations of AutoRound and lm-eval.-| Qwen3-8B W2G64 | Avg. | arc_challenge | hellaswag | gsm8k | lambada_openai | mmlu | mmlupro | truthfulqa_mc1 | winogrande |-|----------------|--------|---------------|-----------|--------|----------------|--------|---------|----------------|------------|-| origin | 0.4373 | 0.4019 | 0.4437 | 0.4215 | 0.4826 | 0.5474 | 0.263 | 0.3072 | 0.6314 |-| new | 0.4787 | 0.4275 | 0.4516 | 0.5944 | 0.5181 | 0.5773 | 0.2807 | 0.3305 | 0.6496 |+| Qwen3-8B W2G64 | Avg. | arc_challenge | hellaswag | gsm8k | lambada_openai | mmlu | mmlupro | truthfulqa_mc1 | winogrande |+|-------------------|--------|---------------|-----------|--------|----------------|--------|---------|----------------|------------|+| AutoRound | 0.4373 | 0.4019 | 0.4437 | 0.4215 | 0.4826 | 0.5474 | 0.263 | 0.3072 | 0.6314 |+| AutoRound+alg_ext | 0.4787 | 0.4275 | 0.4516 | 0.5944 | 0.5181 | 0.5773 | 0.2807 | 0.3305 | 0.6496 || Llama3.1-8B W2G64 | Avg. | arc_challenge | hellaswag | gsm8k | lambada_openai | mmlu | mmlupro | truthfulqa_mc1 | winogrande ||-------------------|--------|---------------|-----------|--------|----------------|--------|---------|----------------|------------|-| origin | 0.382 | 0.3635 | 0.4562 | 0.1622 | 0.5069 | 0.4411 | 0.1661 | 0.3207 | 0.6393 |-| new | 0.4166 | 0.3712 | 0.4729 | 0.2039 | 0.5946 | 0.4981 | 0.2163 | 0.3011 | 0.6748 |\ No newline at end of file+| AutoRound | 0.382 | 0.3635 | 0.4562 | 0.1622 | 0.5069 | 0.4411 | 0.1661 | 0.3207 | 0.6393 |+| AutoRound+alg_ext | 0.4166 | 0.3712 | 0.4729 | 0.2039 | 0.5946 | 0.4981 | 0.2163 | 0.3011 | 0.6748 |\ No newline at end of filediff --git a/docs/auto_scheme_acc.md b/docs/auto_scheme_acc.mdnew file mode 100644index 000000000..b058838f6--- /dev/null+++ b/docs/auto_scheme_acc.md@@ -0,0 +1,51 @@+We use **lm-eval** for evaluation. For LLaMA, we enabled `add_bos_token` and+`removed @use_kernel_forward_from_hub("RMSNorm")`+in [modeling_llama.py](https://github.com/huggingface/transformers/blob/main/src/transformers/models/llama/modeling_llama.py#L52C1-L52C40)+to stabilize accuracy during evaluation. All other settings follow the default configurations of AutoRound and lm-eval.++We ignore the scale and zp bits in the tables below. The accuracy may change a little as we modified a little of the+implementation. We will rerun all the experiments.++For mxfp experiment, we use fake model while for weight only model we use real model. **No tuning is applied unless explicit stated.+**++*Average accuracy across `lambada_openai`, `hellaswag`, `piqa`, `winogrande`, and `mmlu`.*++### Table 1 MXFP4/8 mixed accuracy.++| Average bits | Llama3.1-8B-I | Qwen2.5-7B-I | Qwen3-8B | Qwen3-32B |+|------------------|----------------|----------------|----------------|----------------|+| **BF16** | 0.7076 (100%) | 0.7075 (100%) | 0.6764 (100%) | 0.7321 (100%) |+| **Pure 4-bit** | 0.6626 (93.6%) | 0.6550 (92.6%) | 0.6316 (93.4%) | 0.6901 (94.3%) |+| **Ours 4.5-bit** | 0.6808 (96.2%) | 0.6776 (95.8%) | 0.6550 (96.8%) | 0.7176 (98.0%) |+| **Ours 5-bit** | 0.6857 (96.9%) | 0.6823 (96.4%) | 0.6594 (97.5%) | 0.7201 (98.3%) |+| **Ours 6-bit** | 0.6975 (98.6%) | 0.6970 (98.5%) | 0.6716 (99.3%) | 0.7303 (99.8%) |++We compare the proposed method against naive layer-wise bit allocation strategies, such as assigning higher+precision to the network’s head((near lm-head) or tailad(close to embedding)) layers, to demonstrate its relative+performance advantages.++### Table 2 Comparison with other recipes at an average of 5 bits of mxfp datatype++| Avg. bits = 5 | Llama3.1-8B-I | Qwen2.5-7B-I | Qwen3-8B |+|-----------------------|-------------------:|-------------------:|-------------------:|+| **Tail layers 8-bit** | 0.6671 (94.3%) | 0.6616 (93.5%) | 0.6410 (94.8%) |+| **Head layers 8-bit** | 0.6657 (94.1%) | 0.6686 (94.5%) | 0.6356 (94.0%) |+| **Ours** | **0.6857 (96.9%)** | **0.6823 (96.4%)** | **0.6594 (97.5%)** |++### Table 3 Comparison with other recipes at an average of 4.5 bits of mxfp datatype++| Avg. bits = 4.5 | Llama3.1-8B-I | Qwen2.5-7B-I | Qwen3-8B |+|-----------------------|-------------------:|-------------------:|-------------------:|+| **Tail layers 8-bit** | 0.6614 (93.5%) | 0.6535 (92.4%) | 0.6373 (94.2%) |+| **Head layers 8-bit** | 0.6568 (92.8%) | 0.6642 (93.9%) | 0.6305 (93.2%) |+| **Ours** | **0.6808 (96.2%)** | **0.6776 (95.5%)** | **0.6550 (95.8%)** |+++### Table4 Comparison with other recipes at an average of 3 bits of W2G128 and W4G128++| Avg. bits = 4.5 | Llama3.1-8B-I | Qwen2.5-7B-I | Qwen3-8B |+|-----------------------|--------------:|-------------:|---------:|+| **Tail layers 4-bit** | 0.6058 | 0.3798 | 0.4536 |+| **Head layers 4-bit** | 0.3198 | 0.3270 | 0.3196 |+| **Ours** | 0.6148 | 0.4058 | 0.4862 |\ No newline at end of filediff --git a/docs/step_by_step.md b/docs/step_by_step.mdindex 21c41862c..d2efdcd72 100644--- a/docs/step_by_step.md+++ b/docs/step_by_step.md@@ -19,6 +19,10 @@ This document presents step-by-step instructions for auto-round llm quantization- [AutoRoundBest recipe](#autoroundbest-recipe)- [AutoRoundLight recipe](#autoroundlight-recipe)- [Recipe recommendation](#recipe-recommendation)+ + [AutoScheme](#autoscheme)+ - [CLI Usage](#cli-usage)+ - [API Usage](#api-usage-1)+ - [Hyperparameters in AutoScheme](#hyperparameters-in-autoscheme)+ [RTN mode](#rtn-mode)+ [GGUF format](#gguf-format)+ [Quantization Costs](#quantization-costs)@@ -116,7 +120,7 @@ AutoRound supports several Schemes:- **W3A16**(bits:3,group_size:128,sym:True,act_bits:16)- **W2A16**(bits:2,group_size:128,sym:True,act_bits:16)- **Mixed bits Weight only**-- **NVFP4**(data_type:nvfp4,act_data_type:nvfp4,static_global_scale,group_size 16)+- **NVFP4**(Experimental feature, recommend exporting to llm-compressor format. data_type:nvfp4,act_data_type:nvfp4,static_global_scale,group_size 16)- **MXFP4**(**Research feature,no real kernel**, data_type:mxfp4,act_data_type:mxfp4,rceil,group_size 32)- **FPW8A16**(**Research feature,no real kernel**, data_type:fp8,act_data_type 16:,group_size 0->per tensor )- **FP8_STATIC**(**Research feature,no real kernel**, data_type:fp8,act_data_type:fp8,group_size -1 ->per channel, act_group_size=0->per tensor)@@ -271,6 +275,61 @@ W2G64 Average Accuracy of 13 tasks and Time Cost Results(Testing was conducted o</details>+### AutoScheme++AutoScheme provide automatically algorithm to provide mixed bits/data_type quantization recipes. For some accuracy result, please refer this doc [here](./auto_scheme_acc.md)++**Please note that mixed data types are supported during tuning, but cannot be exported to real models at this time..**+### CLI Usage+use `iters=200`for tuning.+~~~bash+auto_round \+ --model_name $model_name \+ --avg_bits 6 \+ --options "mxfp4,mxfp8" \+ --ignore_scale_zp_bits \+ --iters 0 \+ --format fake+~~~++### API Usage+~~~+avg_bits= 3.0+scheme = AutoScheme(avg_bits=avg_bits, options=("W2A16G64“, "W4A16","W8A16"))+ar = AutoRound(model=model_name, scheme=scheme, iters=0, nsamples=1)+ar.quantize_and_save()+~~~++### Hyperparameters in AutoScheme+`avg_bits(float)`: Target average bits for the whole model, only to be quantized layer will be counted in the average bits calculation.++`options(Union[str, list[Union[QuantizationScheme, str]])`: the options of quantization schemes to choose from. It could be a string like "W4A16", or a list of strings or QuantizationScheme objects.++`ignore_scale_zp_bits(bool)`: Whether to ignore the bits of scale and zero point in average bits calculation. Default is False.++`shared_layers (Optional[Iterable[Iterable[str]]])` only supported in API now++`device_map (Optional[str,dict,torch.device])` only supported in API now, as auto-scheme used more VRAM than auto-round tuning, so you could set a different device_map for it.++In some serving frameworks, certain layers (e.g., QKV or MoE) are fused to accelerate inference. These fused layers may require the same data type and bit configuration. The shared_layers option simplifies this setup by supporting both regex and full-name matching. **Note that regex matching is applied in a block-wise manner.**+++```python+from auto_round import AutoRound, AutoScheme++shared_layers = [+ ["*.self_attn.k_proj", "v_proj", "q_proj", "out_proj"],+ ("model.decoder.layers.6.fc1", "model.decoder.layers.6.fc2"),+ ("fc1", "fc2"),+]+target_bits = 5.0+model_name = "facebook/opt-125m"+scheme = AutoScheme(avg_bits=target_bits, options=("W4A16", "MXFP8"), shared_layers=shared_layers)+ar = AutoRound(model=model_name, scheme=scheme, iters=0, nsamples=1)+model, layer_config = ar.quantize()+```++### RTN modeAutoRound also supports RTN (Round-To-Nearest) mode for fast, calibration-free baseline quantization. try setting `iters=0` and use `group_size=32` for better results.diff --git a/test/test_cpu/test_auto_scheme.py b/test/test_cpu/test_auto_scheme.pynew file mode 100644index 000000000..a0eb21001--- /dev/null+++ b/test/test_cpu/test_auto_scheme.py@@ -0,0 +1,25 @@+import shutil+import sys+import unittest++sys.path.insert(0, "../..")+from auto_round import AutoRound, AutoRoundConfig, AutoScheme+++class TestAutoScheme(unittest.TestCase):+ @classmethod+ def setUpClass(self):+ self.save_dir = "./saved"+ self.tasks = "lambada_openai"++ @classmethod+ def tearDownClass(self):+ shutil.rmtree("./saved", ignore_errors=True)+ shutil.rmtree("runs", ignore_errors=True)++ def test_auto_scheme_export(self):+ model_name = "/tf_dataset/auto_round/models/facebook/opt-125m"+ scheme = AutoScheme(avg_bits=2, options=("W2A16"), nsamples=1, ignore_scale_zp_bits=True)+ ar = AutoRound(model=model_name, scheme=scheme)+ ar.quantize_and_save(self.save_dir)+ shutil.rmtree(self.save_dir, ignore_errors=True)diff --git a/test/test_cpu/test_autoround.py b/test/test_cpu/test_autoround.pyindex de5f7412a..cbd0583df 100644--- a/test/test_cpu/test_autoround.py+++ b/test/test_cpu/test_autoround.py@@ -736,6 +736,7 @@ class TestAutoRound(unittest.TestCase):iters=1,layer_config=layer_config,)+ ar.quantize()def test_quant_lm_head(self):model_name = "/tf_dataset/auto_round/models/Qwen/Qwen3-8B"diff --git a/test/test_cuda/test_auto_scheme.py b/test/test_cuda/test_auto_scheme.pynew file mode 100644index 000000000..5b7116768--- /dev/null+++ b/test/test_cuda/test_auto_scheme.py@@ -0,0 +1,192 @@+import copy+import re+import shutil+import sys+import unittest++sys.path.insert(0, "../..")+from auto_round import AutoRound, AutoRoundConfig, AutoScheme+from auto_round.auto_scheme.utils import compute_avg_bits_for_model+from auto_round.eval.evaluation import simple_evaluate+from auto_round.testing_utils import multi_card+from auto_round.utils import get_module+++class TestAutoScheme(unittest.TestCase):+ @classmethod+ def setUpClass(self):+ self.save_dir = "./saved"+ self.tasks = "lambada_openai"++ @classmethod+ def tearDownClass(self):+ shutil.rmtree("./saved", ignore_errors=True)+ shutil.rmtree("runs", ignore_errors=True)++ def test_gguf_export(self):+ model_name = "/models/Qwen3-0.6B"+ target_bits = 3+ scheme = AutoScheme(avg_bits=target_bits, options=("GGUF:Q2_K_S", "GGUF:Q4_K_M"), ignore_scale_zp_bits=True)+ ar = AutoRound(model=model_name, scheme=scheme, iters=0)+ ar.quantize_and_save(self.save_dir, format="gguf:q2_k_s")+ shutil.rmtree(self.save_dir, ignore_errors=True)++ def test_gguf(self):+ model_name = "/models/Qwen3-8B"+ target_bits = 3+ scheme = AutoScheme(avg_bits=target_bits, options=("GGUF:Q2_K_S", "GGUF:Q4_K_M"), ignore_scale_zp_bits=True)+ ar = AutoRound(model=model_name, scheme=scheme, iters=0, nsamples=1, disable_opt_rtn=True)+ model, layer_config = ar.quantize()+ avg_bits, _ = compute_avg_bits_for_model(model, ignore_scale_zp_bits=True)+ print(avg_bits)+ assert target_bits - 0.1 < avg_bits <= target_bits + 1e-3++ def test_shared_layers(self):+ model_name = "/models/opt-125m"+ from transformers import AutoModelForCausalLM, AutoTokenizer++ model = AutoModelForCausalLM.from_pretrained(model_name)+ shared_layers = [+ ["*.self_attn.k_proj", "v_proj", "q_proj", "out_proj"],+ ("model.decoder.layers.6.fc1", "model.decoder.layers.6.fc2"),+ ("fc1", "fc2"),+ ]+ from auto_round.auto_scheme.utils import parse_shared_layers++ res = parse_shared_layers(model, shared_layers)+ self.assertEqual(len(res), 24)+ assert [+ "model.decoder.layers.2.self_attn.out_proj",+ "model.decoder.layers.2.self_attn.q_proj",+ "model.decoder.layers.2.self_attn.v_proj",+ ] in res+ assert ["model.decoder.layers.6.fc1", "model.decoder.layers.6.fc2"] in res+ assert ["model.decoder.layers.7.fc1", "model.decoder.layers.7.fc2"] in res+ target_bits = 5.0+ scheme = AutoScheme(avg_bits=target_bits, options=("W4A16", "MXFP8"), shared_layers=shared_layers)+ ar = AutoRound(model=model_name, scheme=scheme, iters=0, nsamples=1)+ model, layer_config = ar.quantize()+ avg_bits, _ = compute_avg_bits_for_model(model)+ for names in res:+ bits = []+ for name in names:+ module = get_module(model, name)+ if hasattr(module, "orig_layer"):+ bits.append(module.orig_layer.bits)+ else:+ bits.append(module.bits)+ bits = set(bits)+ self.assertEqual(len(bits), 1)+ print(avg_bits)+ assert target_bits - 0.1 < avg_bits <= target_bits + 1e-3++ #+ @multi_card+ def test_multi_card(self):+ model_name = "/models/Qwen3-8B"+ target_bits = 5.254+ # for device_map in ["auto", "0,1", "0", None]:+ scheme = AutoScheme(avg_bits=target_bits, options=("NVFP4"))+ ar = AutoRound(model=model_name, scheme=scheme, iters=0, nsamples=1)+ model, layer_config = ar.quantize()+ avg_bits, _ = compute_avg_bits_for_model(model)+ print(avg_bits)+ assert target_bits - 0.1 < avg_bits <= target_bits + 1e-3++ @multi_card+ def test_dict_device_map(self):+ model_name = "/models/Qwen3-8B"+ target_bits = 8.755+ device_map = {"up_proj": 0, "down_proj": 1}++ scheme = AutoScheme(avg_bits=target_bits, options=("MXFP8"))+ ar = AutoRound(model=model_name, scheme=scheme, iters=0, nsamples=1, device_map=device_map)+ model, layer_config = ar.quantize()+ avg_bits, _ = compute_avg_bits_for_model(model)+ print(avg_bits)+ assert target_bits - 0.1 < avg_bits <= target_bits + 1e-3++ def test_min_target_bits(self):+ model_name = "/models/opt-125m"+ target_bits = 4.644+ scheme = AutoScheme(avg_bits=target_bits, options=("MXFP4", "W8A16"))+ ar = AutoRound(model=model_name, scheme=scheme, iters=0, nsamples=1)+ model, layer_config = ar.quantize()+ avg_bits, _ = compute_avg_bits_for_model(model)+ print(avg_bits)+ assert target_bits - 0.1 < avg_bits <= target_bits + 1e-3++ #+ def test_max_target_bits(self):+ model_name = "/models/opt-125m"+ target_bits = 8.211+ scheme = AutoScheme(avg_bits=target_bits, options=("MXFP4", "W8A16"))+ ar = AutoRound(model=model_name, scheme=scheme, iters=0, nsamples=1)+ model, layer_config = ar.quantize()+ avg_bits, _ = compute_avg_bits_for_model(model)+ print(avg_bits)+ assert target_bits - 0.1 < avg_bits <= target_bits + 1e-3++ def test_patch_scheme(self):+ model_name = "/models/opt-125m"+ target_bits = 5+ scheme = AutoScheme(avg_bits=target_bits, options=("MXFP4", "W8A16"))+ ar = AutoRound(model=model_name, scheme=scheme, iters=0, nsamples=1, group_size=32)+ model, layer_config = ar.quantize()+ for n, m in model.named_modules():+ if hasattr(m, "group_size"):+ self.assertEqual(m.group_size, 32)+ avg_bits, _ = compute_avg_bits_for_model(model)+ print(avg_bits)+ assert target_bits - 0.1 < avg_bits <= target_bits + 1e-3++ def test_layer_config(self):+ target_bits = 3.0+ model_name = "/models/opt-125m"+ scheme = AutoScheme(avg_bits=3, options=("W2A16", "W4A16", "BF16"))+ user_layer_config = {"model.decoder.layers.10.fc1": {"bits": 8, "group_size": 32, "sym": False}}+ ar = AutoRound(model=model_name, scheme=scheme, iters=0, nsamples=1, layer_config=user_layer_config)+ model, layer_config = ar.quantize()+ self.assertEqual(layer_config["model.decoder.layers.10.fc1"]["bits"], 8)+ self.assertEqual(layer_config["model.decoder.layers.10.fc1"]["sym"], False)+ self.assertEqual(layer_config["model.decoder.layers.10.fc1"]["group_size"], 32)+ layer = get_module(model, "model.decoder.layers.10.fc1")+ self.assertEqual(layer.bits, 8)+ self.assertEqual(layer.sym, False)+ self.assertEqual(layer.group_size, 32)+ avg_bits, _ = compute_avg_bits_for_model(model)+ print(avg_bits)+ assert target_bits - 0.1 < avg_bits <= target_bits + 1e-3++ def test_lm_head_and_mix_dtype(self):+ model_name = "/models/Qwen3-8B"+ target_bits = 6+ scheme = AutoScheme(avg_bits=target_bits, options=("MXFP4", "MXFP8"))+ ar = AutoRound(model=model_name, scheme=scheme, iters=0, nsamples=1, quant_lm_head=True)+ model, layer_config = ar.quantize()+ self.assertLessEqual(layer_config["lm_head"]["bits"], 8)+ avg_bits, _ = compute_avg_bits_for_model(model)+ print(avg_bits)+ assert target_bits - 0.1 < avg_bits <= target_bits + 1e-3++ def test_auto_scheme_export(self):+ model_name = "/models/opt-125m"+ scheme = AutoScheme(avg_bits=3, options=("W2A16", "W4A16", "W8A16", "BF16"))+ ar = AutoRound(model=model_name, scheme=scheme)+ ar.quantize_and_save(self.save_dir)+ model_args = f"pretrained={self.save_dir}"+ result = simple_evaluate(model="hf", model_args=model_args, tasks="lambada_openai", batch_size="auto")+ print(result["results"]["lambada_openai"]["acc,none"])+ self.assertGreater(result["results"]["lambada_openai"]["acc,none"], 0.25)+ shutil.rmtree(self.save_dir, ignore_errors=True)++ def test_enable_torch_compile(self):+ model_name = "/models/opt-125m"+ scheme = AutoScheme(avg_bits=2, options=("W2A16"), ignore_scale_zp_bits=True)+ ar = AutoRound(model=model_name, scheme=scheme, enable_torch_compile=True)+ ar.quantize_and_save(self.save_dir)+ model_args = f"pretrained={self.save_dir}"+ result = simple_evaluate(model="hf", model_args=model_args, tasks="lambada_openai", batch_size="auto")+ print(result["results"]["lambada_openai"]["acc,none"])+ self.assertGreater(result["results"]["lambada_openai"]["acc,none"], 0.10)+ shutil.rmtree(self.save_dir, ignore_errors=True)