PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度
enable regex quantization config saving for mixed bits (#825)
Weiwei873114a0ff5e
diff --git a/auto_round/compressors/base.py b/auto_round/compressors/base.pyindex d948de65b..f1c6c5bd4 100644--- a/auto_round/compressors/base.py+++ b/auto_round/compressors/base.py@@ -439,7 +439,7 @@ class BaseCompressor(object):"Please save the model using the `fake` format for now.")- layer_config, self.has_qlayer_outside_block = set_layer_config(+ layer_config, self.has_qlayer_outside_block, self.regex_config = set_layer_config(self.model,self.layer_config,self.scheme,@@ -1653,7 +1653,7 @@ class BaseCompressor(object):enable_gguf_official_mixed = Trueelse:enable_gguf_official_mixed = False- self.layer_config, self.has_qlayer_outside_block = set_layer_config(+ self.layer_config, self.has_qlayer_outside_block, self.regex_config = set_layer_config(self.model,self.layer_config,self.scheme,@@ -2937,6 +2937,8 @@ class BaseCompressor(object):"Support for exporting activation quantization is limited. ""Please ensure that your configuration is supported.")+ # if format == "llm_compressor" and (is_nv_fp(self.data_type) or is_mx_fp(self.data_type)):+ # format = format.replace("llm_compressor", f"llm_compressor:{self.data_type}")if format == "llm_compressor" and (is_nv_fp(self.data_type) or is_mx_fp(self.data_type)):format = format.replace("llm_compressor", f"llm_compressor:{self.data_type}")if format == "llm_compressor" and is_static_wfp8afp8(self):@@ -2985,6 +2987,7 @@ class BaseCompressor(object):"act_data_type","super_bits","super_group_size",+ "regex_config",]if isinstance(self.dataset, str):serialization_keys.append("dataset")diff --git a/auto_round/export/export_to_autogptq/export.py b/auto_round/export/export_to_autogptq/export.pyindex 7e8497441..5c795bac9 100644--- a/auto_round/export/export_to_autogptq/export.py+++ b/auto_round/export/export_to_autogptq/export.py@@ -17,6 +17,8 @@ import inspectimport jsonimport osfrom concurrent.futures import ThreadPoolExecutor+from dataclasses import fields+from typing import Any, Dictimport threadpoolctl as tctl@@ -48,16 +50,28 @@ from tqdm import tqdmimport auto_round.export.export_to_autogptq.qlinear_tritonfrom auto_round.export.utils import save_model+from auto_round.schemes import QuantizationScheme++GPTQ_REQUIRED_CONFIG_KEYS = (+ "bits",+ "group_size",+ "sym",+)+from auto_round.logger import loggerfrom auto_round.utils import (SUPPORTED_LAYER_TYPES,+ check_start_with_block_name,check_to_quantized,copy_python_files_from_model_cache,filter_quantization_config,get_autogptq_packing_qlinear,get_block_names,get_module,+ json_serialize,+ matches_any_regex,set_module,+ to_standard_regex,)BLOCK_PATTERNS = [ ## copy from transformers optimum@@ -66,6 +80,54 @@ BLOCK_PATTERNS = [ ## copy from transformers optimum"gpt_neox.layers","model.layers",]+from auto_round.export.export_to_autoround.utils import check_neq_config+++def convert_to_autogptq_dynamic(regex_config: Dict[str, Dict[str, Any]]) -> Dict[str, Dict[str, Any]]:+ """+ Convert AutoRound-style regex_config into AutoGPTQ-style QuantizerConfig.dynamic.++ Rules:+ - bits < 16 -> quantize -> positive match `+:regex`+ - bits == 16 -> skip quantize -> negative match `-:regex`+ """+ converted = {}+ for name, cfg in regex_config.items():+ bits = cfg.get("bits")+ regex = to_standard_regex(name)++ if bits is None:+ continue # ignore invalid entries+ elif bits < 16:+ converted[f"+:{regex}"] = {"bits": bits}+ for key in GPTQ_REQUIRED_CONFIG_KEYS: # only save keys gptq supported+ converted[f"+:{regex}"][key] = regex_config[name][key]+ else:+ # skip quantization+ converted[f"-:{regex}"] = {}+ return converted+++def convert_from_autogptq_dynamic(dynamic_config: dict) -> dict:+ """+ Convert AutoGPTQ-style QuantizerConfig.dynamic into AutoRound-style extra_config.++ Rules:+ - '+:regex' => quantize => keep bits and other quantization keys+ - '-:regex' => skip quantize => set bits to 16 (FP16 passthrough)+ """+ converted = {}+ for name, cfg in dynamic_config.items():+ # Strip the +: or -:+ if name.startswith("+:"):+ regex = name[2:]+ # keep all config fields (bits, group_size, sym, etc.)+ converted[regex] = dict(cfg)+ elif name.startswith("-:"):+ regex = name[2:]+ # mark skipped layers with bits=16+ converted[regex] = {"bits": 16, "act_bits": 16}+ return converteddef pack_layer(name, model, backend, device=None):@@ -132,58 +194,93 @@ def pack_layer(name, model, backend, device=None):def save_quantized_as_autogptq(output_dir, inplace=True, backend="auto_gptq:exllamav2", **kwargs):"""Export the model to autogptq format to easily leverage cuda kernel."""+ # --- 1️⃣ Extract inputs & configs ---model = kwargs["model"]- safe_serialization = True if "safe_serialization" not in kwargs.keys() else kwargs["safe_serialization"]- quant_block_list = kwargs.get("quant_block_list", get_block_names(model))- tokenizer = kwargs.get("tokenizer", None)- processor = kwargs.get("processor", None)- device = kwargs.get("device", None)- image_processor = kwargs.get("image_processor", None)- if output_dir is not None and os.path.exists(output_dir):- logger.warning(f"{output_dir} already exists, this may cause model conflict")- if output_dir is not None and tokenizer is not None and hasattr(tokenizer, "save_pretrained"):- tokenizer.save_pretrained(output_dir)- if output_dir is not None and processor is not None:- processor.save_pretrained(output_dir)- if output_dir is not None and image_processor is not None:- image_processor.save_pretrained(output_dir)- ##check module quantized in block, this may have bug for mixed precision quantizationquantization_config = kwargs["serialization_dict"]+ layer_config = kwargs["layer_config"]+ quant_block_list = kwargs.get("quant_block_list", get_block_names(model))+ tokenizer = kwargs.get("tokenizer")+ processor = kwargs.get("processor")+ image_processor = kwargs.get("image_processor")+ device = kwargs.get("device")+ safe_serialization = kwargs.get("safe_serialization", True)++ # --- Save metadata (tokenizer, processor, etc.) ---+ if output_dir:+ if os.path.exists(output_dir):+ logger.warning(f"{output_dir} already exists, may cause overwrite conflicts.")+ for comp in (tokenizer, processor, image_processor):+ if comp is not None and hasattr(comp, "save_pretrained"):+ comp.save_pretrained(output_dir)++ # --- Handle quantization structure ---all_blocks = quant_block_list- flattened_list = [item for sublist in all_blocks for item in sublist]- common_prefix = os.path.commonprefix(flattened_list).rstrip(".")- if common_prefix not in BLOCK_PATTERNS:- logger.error("auto-gptq format may not support loading this quantized model")+ flattened = [x for sub in all_blocks for x in sub]+ common_prefix = os.path.commonprefix(flattened).rstrip(".")++ if "BLOCK_PATTERNS" in kwargs and common_prefix not in kwargs["BLOCK_PATTERNS"]:+ logger.error(f"Unsupported block prefix '{common_prefix}' for AutoGPTQ format.")quantization_config["block_name_to_quantize"] = common_prefixquantization_config.pop("to_quant_block_names", None)- ## as layers maybe already packed, we need to check in layer_config- layer_config = kwargs["layer_config"]+ # --- Build per-layer dynamic overrides ---+ regex_config = quantization_config.pop("regex_config", {})+ block_name_to_quantize = quantization_config.get("block_name_to_quantize")+ extra_config = {}+ lm_head_quantized = False+ scheme_keys = [f.name for f in fields(QuantizationScheme)]+ for layer_name, cfg in layer_config.items():+ bits = cfg.get("bits", 16)+ in_blocks = cfg.get("in_blocks", False)+ # Handle non-block layers (e.g., LM head)+ if not in_blocks and bits <= 8:+ lm_head_quantized = True+ extra_config[layer_name] = {k: cfg[k] for k in GPTQ_REQUIRED_CONFIG_KEYS}+ continue+ # Handle block layers+ if in_blocks or (block_name_to_quantize and check_start_with_block_name(layer_name, block_name_to_quantize)):+ neq_keys = check_neq_config(cfg, **{k: quantization_config[k] for k in scheme_keys})+ if neq_keys:+ if matches_any_regex(layer_name, regex_config):+ continue+ extra_config[layer_name] = {k: cfg[k] for k in GPTQ_REQUIRED_CONFIG_KEYS}++ # --- Merge regex_config + extra_config into GPTQ dynamic config ---+ dynamic = {}+ if regex_config:+ dynamic.update(convert_to_autogptq_dynamic(regex_config))+ if extra_config:+ dynamic.update(convert_to_autogptq_dynamic(extra_config))+ if dynamic:+ quantization_config["dynamic"] = dynamic++ # --- Block-wise quantization verification ---for n, m in model.named_modules():m.tmp_name = nall_to_quantized = Truemodules_in_block_to_quantize = []- for block_names in all_blocks:- first_block = get_module(model, block_names[0])- for n, m in first_block.named_modules():- if m.tmp_name not in layer_config.keys():- continue- if not check_to_quantized(layer_config[m.tmp_name]):- all_to_quantized = False- else:- modules_in_block_to_quantize.append(n)- modules_in_block_to_quantize = [modules_in_block_to_quantize]+ if not dynamic: # Only uniform precision+ for block_names in all_blocks:+ first_block = get_module(model, block_names[0])+ for n, m in first_block.named_modules():+ if m.tmp_name not in layer_config:+ continue+ if not check_to_quantized(layer_config[m.tmp_name]):+ all_to_quantized = False+ else:+ modules_in_block_to_quantize.append(n)+ modules_in_block_to_quantize = [modules_in_block_to_quantize]+if all_to_quantized:modules_in_block_to_quantize = None- for n, m in model.named_modules():+ for _, m in model.named_modules():delattr(m, "tmp_name")if not inplace:model = copy.deepcopy(model.to("cpu"))- layer_config = kwargs["layer_config"]names = list(layer_config.keys())max_workers = 1if not torch.cuda.is_available() and not torch.xpu.is_available():@@ -202,6 +299,7 @@ def save_quantized_as_autogptq(output_dir, inplace=True, backend="auto_gptq:exllpassif output_dir is None:return model+ quantization_config["lm_head"] = lm_head_quantizedquantization_config["provider"] = "auto-round"quantization_config["quant_method"] = "gptq"quantization_config.pop("dataset", None) ## pile-10k is not supported in gptqdiff --git a/auto_round/export/export_to_autoround/export.py b/auto_round/export/export_to_autoround/export.pyindex b31b81a16..679257762 100644--- a/auto_round/export/export_to_autoround/export.py+++ b/auto_round/export/export_to_autoround/export.py@@ -44,6 +44,7 @@ from auto_round.utils import (is_nv_fp,is_standard_fp,set_module,+ to_standard_regex,)@@ -340,8 +341,15 @@ def save_quantized_as_autoround(output_dir, inplace=True, backend="auto_round:exif cfg[key] is not None:extra_config[layer_name][key] = cfg[key]+ regex_config = quantization_config.pop("regex_config")+ if regex_config is not None:+ for name in regex_config.keys():+ regex_name = to_standard_regex(name)+ extra_config[regex_name] = {**{k: regex_config[name][k] for k in scheme_keys}}+if len(extra_config) > 0:quantization_config["extra_config"] = extra_config+names = list(layer_config.keys())max_workers = 1if not torch.cuda.is_available() and not torch.xpu.is_available():diff --git a/auto_round/export/export_to_autoround/export_to_nvfp_mxfp.py b/auto_round/export/export_to_autoround/export_to_nvfp_mxfp.pyindex 9e3a73533..e344b51b3 100644--- a/auto_round/export/export_to_autoround/export_to_nvfp_mxfp.py+++ b/auto_round/export/export_to_autoround/export_to_nvfp_mxfp.py@@ -41,6 +41,7 @@ from auto_round.utils import (is_nv_fp,set_amax_for_all_moe_layers,set_module,+ to_standard_regex,)from auto_round.wrapper import WrapperWALayer@@ -211,6 +212,12 @@ def save_quantized_as_fp(output_dir, inplace=True, **kwargs):if cfg[key] is not None:extra_config[layer_name][key] = cfg[key]+ regex_config = quantization_config.pop("regex_config")+ if regex_config is not None:+ for name in regex_config.keys():+ regex_name = to_standard_regex(name)+ extra_config[regex_name] = {**{k: regex_config[name][k] for k in scheme_keys}}+if len(extra_config) > 0:quantization_config["extra_config"] = extra_confignames = list(layer_config.keys())diff --git a/auto_round/export/export_to_awq/export.py b/auto_round/export/export_to_awq/export.pyindex a1f2d4d26..2cf256c67 100644--- a/auto_round/export/export_to_awq/export.py+++ b/auto_round/export/export_to_awq/export.py@@ -137,6 +137,7 @@ def save_quantized_as_autoawq(output_dir, inplace=True, **kwargs):return modelquantization_config = kwargs["serialization_dict"]+ regex_config = quantization_config.pop("regex_config", {}) # awq do not support mixed bits config savingif output_dir is None:return compressed_model@@ -145,11 +146,16 @@ def save_quantized_as_autoawq(output_dir, inplace=True, **kwargs):for key in layer_config.keys():if not check_to_quantized(layer_config[key]) and not any(name in key for name in modules_to_not_convert):modules_to_not_convert.append(key)+ for key, cfg in regex_config.items():+ bits = cfg.get("bits")+ if bits > 8: # save fp_layer regexs+ modules_to_not_convert.append(key)+quantization_config["provider"] = "auto-round"quantization_config["quant_method"] = "awq"quantization_config["zero_point"] = not quantization_config["sym"]quantization_config["version"] = "gemm"- quantization_config["modules_to_not_convert"] = modules_to_not_convert+ quantization_config["modules_to_not_convert"] = list(dict.fromkeys(modules_to_not_convert))##check module quantized in block, this may have bug for mixed precision quantizationfilter_quantization_config(quantization_config)if hasattr(compressed_model, "config"):diff --git a/auto_round/export/export_to_llmcompressor/export_to_fp.py b/auto_round/export/export_to_llmcompressor/export_to_fp.pyindex 633593075..f5c273856 100644--- a/auto_round/export/export_to_llmcompressor/export_to_fp.py+++ b/auto_round/export/export_to_llmcompressor/export_to_fp.py@@ -25,6 +25,7 @@ import transformersfrom tqdm import tqdmfrom auto_round.export.export_to_autoround.qlinear_fp import QuantLinear+from auto_round.export.export_to_llmcompressor.utils import generate_ignore_regex_listfrom auto_round.export.utils import save_modelfrom auto_round.logger import loggerfrom auto_round.utils import (@@ -114,9 +115,8 @@ def pack_layer(name, model, backend, device=None):scale = layer.scaleglobal_scale = getattr(layer, "weight_global_scale", None)input_global_scale = getattr(layer, "input_global_scale", None)- # zero = layer.zp+ # zero = layer.zp # no zeros to handle, as mxfp not support asym quantizationqlayer.pack(layer, scale, global_scale=global_scale, input_global_scale=input_global_scale, device=device)- ## no zeros to handle, as mxfp not support asym quantizationqlayer.to(orig_device)@@ -155,6 +155,9 @@ def save_quantized_as_fp(output_dir, inplace=True, **kwargs):device = kwargs.get("device", None)tokenizer = kwargs.get("tokenizer", None)processor = kwargs.get("processor", None)+ ar_quantization_config = kwargs["serialization_dict"]+ regex_config = ar_quantization_config.pop("regex_config")+ layer_config = kwargs["layer_config"]extra_config = {}if act_bits <= 8:@@ -199,12 +202,7 @@ def save_quantized_as_fp(output_dir, inplace=True, **kwargs):for _ in executor.map(wrapper, names):pass- # TODO fix the ignore re match issue, compile with fp8 & int8 config- ignore = ["lm_head"]- for layer_name in layer_config:- if layer_config[layer_name]["bits"] > 8: ## find ignore layers- ignore.append(layer_name)- ignore = list(set(ignore))+ ignore = generate_ignore_regex_list(regex_config=regex_config, layer_config=layer_config)# get llm-compressor format configcheck_compressed_tensors_supported()diff --git a/auto_round/export/export_to_llmcompressor/utils.py b/auto_round/export/export_to_llmcompressor/utils.pynew file mode 100644index 000000000..304a720bd--- /dev/null+++ b/auto_round/export/export_to_llmcompressor/utils.py@@ -0,0 +1,51 @@+# Copyright (c) 2025 Intel Corporation+#+# Licensed under the Apache License, Version 2.0 (the "License");+# you may not use this file except in compliance with the License.+# You may obtain a copy of the License at+#+# http://www.apache.org/licenses/LICENSE-2.0+#+# Unless required by applicable law or agreed to in writing, software+# distributed under the License is distributed on an "AS IS" BASIS,+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+# See the License for the specific language governing permissions and+# limitations under the License.++from typing import Dict, List++from auto_round.utils import matches_any_regex, to_standard_regex+++def generate_ignore_regex_list(regex_config: Dict[str, Dict], layer_config: Dict[str, Dict]) -> List[str]:+ """+ Generate ignore regex list for llm_compressor based on regex_config and layer_config.++ Rules:+ 1. Any layer in regex_config with bits >= 16 is ignored.+ 2. Any layer in layer_config with bits >= 16 is ignored if not already included.+ 3. Output regex patterns are normalized for llm_compressor ('re:...' style).++ Args:+ regex_config (Dict[str, Dict]): dynamic quantization config+ layer_config (Dict[str, Dict]): layer-wise quantization config++ Returns:+ List[str]: List of regex patterns to ignore during quantization.+ """+ prefix = "re:"+ ignore_regex: List[str] = []++ # Step 1: Add regex_config keys with bits >= 16+ for key, cfg in regex_config.items():+ bits = cfg.get("bits")+ if bits > 8:+ ignore_regex.append(prefix + to_standard_regex(key))++ # Step 2: Add all full named layer from layer_config if bits >= 16+ for key, cfg in layer_config.items():+ bits = cfg.get("bits")+ if bits > 8:+ ignore_regex.append(key)++ return ignore_regexdiff --git a/auto_round/inference/convert_model.py b/auto_round/inference/convert_model.pyindex 2c27c0374..a01623f73 100644--- a/auto_round/inference/convert_model.py+++ b/auto_round/inference/convert_model.py@@ -28,6 +28,7 @@ from auto_round.inference.backend import (get_layer_backend,process_requirement,)+from auto_round.inference.utils import _expand_regex_configfrom auto_round.logger import loggerfrom auto_round.schemes import QuantizationSchemefrom auto_round.utils import (@@ -258,21 +259,39 @@ def get_layer_config(model, quantization_config):if not any([re.search(re.compile(n), layer_name) is not None for n in modules_in_block_to_quantize]):extra_config[layer_name] = {"bits": 16} # Default to 16-bit for unquantized layers- # Process AWQ format: exclude specified modules from quantization+ # Expand GPTQ 'dynamic' config (regex-based)+ dynamic_config = getattr(quantization_config, "dynamic", None)+ from auto_round.export.export_to_autogptq.export import convert_from_autogptq_dynamic++ if dynamic_config and isinstance(dynamic_config, dict):+ extra_config = _expand_regex_config(+ regex_config=convert_from_autogptq_dynamic(dynamic_config),+ base_config=extra_config,+ layer_names=layer_names,+ model=model,+ )++ # AWQ format: exclude specified modulesextra_config = skip_not_convert_modules(model, quantization_config, layer_names, extra_config)- # Merge and deduplicate layer names+ # Expand auto_round regex configs (regex-based)+ extra_config = _expand_regex_config(+ regex_config=extra_config, base_config=extra_config, layer_names=layer_names, model=model+ )++ # Merge and deduplicatelayer_names = list(set(layer_names).union(extra_config.keys()))- # Construct final layer configuration+ # Build final layer configslayer_configs = {}quant_scheme_attrs = QuantizationScheme.get_attributes()for layer_name in layer_names:- layer_config = {}- layer_extra_config = extra_config.get(layer_name, {})- for scheme_attr in quant_scheme_attrs:- layer_config[scheme_attr] = layer_extra_config.get(scheme_attr, getattr(default_quant_scheme, scheme_attr))- layer_configs[layer_name] = QuantizationScheme.from_dict(layer_config)+ layer_cfg_dict = {}+ layer_extra = extra_config.get(layer_name, {})+ for attr in quant_scheme_attrs:+ layer_cfg_dict[attr] = layer_extra.get(attr, getattr(default_quant_scheme, attr))+ layer_configs[layer_name] = QuantizationScheme.from_dict(layer_cfg_dict)+return layer_configsdiff --git a/auto_round/inference/utils.py b/auto_round/inference/utils.pynew file mode 100644index 000000000..fdfa44161--- /dev/null+++ b/auto_round/inference/utils.py@@ -0,0 +1,62 @@+# Copyright (c) 2025 Intel Corporation+#+# Licensed under the Apache License, Version 2.0 (the "License");+# you may not use this file except in compliance with the License.+# You may obtain a copy of the License at+#+# http://www.apache.org/licenses/LICENSE-2.0+#+# Unless required by applicable law or agreed to in writing, software+# distributed under the License is distributed on an "AS IS" BASIS,+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+# See the License for the specific language governing permissions and+# limitations under the License.+import re++from auto_round.utils import SUPPORTED_LAYER_TYPES+++def _expand_regex_config(regex_config, base_config, layer_names, model):+ """+ Expand regex-based layer configs to full layer names.++ Args:+ regex_config (dict): regex-based config (dynamic_config or part of extra_config)+ base_config (dict): extra_config to write into+ layer_names (list): known quantization layer names+ model (nn.Module): target model++ Returns:+ dict: expanded base_config+ """+ if not regex_config:+ return base_config++ # Collect all supported layer names in model+ all_supported_layer_names = [n for n, m in model.named_modules() if isinstance(m, SUPPORTED_LAYER_TYPES)]++ # Identify which keys are regex patterns (not exact layer names)+ regex_keys = [k for k in regex_config.keys() if k not in all_supported_layer_names]++ for regex_key in regex_keys:+ try:+ pattern = re.compile(regex_key)+ except re.error:+ # invalid regex, skip silently+ continue++ # Prefer matches within layer_names first+ matched_layers = [ln for ln in layer_names if re.search(pattern, ln)]+ if not matched_layers:+ matched_layers = [ln for ln in all_supported_layer_names if re.search(pattern, ln)]++ if matched_layers:+ cfg = regex_config[regex_key]+ if cfg == {}:+ continue+ for ln in matched_layers:+ # do not overwrite explicit layer config+ if ln not in base_config:+ base_config[ln] = cfg++ return base_configdiff --git a/auto_round/utils.py b/auto_round/utils.pyindex c73d6a210..84ecadd76 100644--- a/auto_round/utils.py+++ b/auto_round/utils.py@@ -25,7 +25,7 @@ from dataclasses import asdict, fieldsfrom enum import Enumfrom functools import lru_cachefrom pathlib import Path-from typing import Any, Callable, Tuple, Union+from typing import Any, Callable, Dict, List, Tuple, Unionimport cpuinfoimport torch@@ -2828,7 +2828,7 @@ def set_layer_config(quant_lm_head: bool = False,enable_gguf_official_mixed: bool = True,is_mllm: bool = False,-) -> tuple[dict, bool]:+) -> tuple[dict, bool, dict]:"""Normalize, validate, and expand layer-specific quantization configs.Returns (final_layer_config, has_quant_layer_outside_block)@@ -2923,6 +2923,7 @@ def set_layer_config(embedding_layer_names.append(n)# 6. expand regex configs+ regex_config = {}for name in list(layer_config.keys()):if name in all_supported_layer_names:continue@@ -2938,8 +2939,10 @@ def set_layer_config(if not matched:raise ValueError(f"Invalid '{name}' in layer_config, no match found.")val = layer_config.pop(name)+ regex_config[name] = val # keep regex configfor match in matched:layer_config[match] = val+ # regex_config = None if len(regex_config)==0 else regex_config# 7. lm_headlm_head_name = get_lm_head_name(model)@@ -2996,7 +2999,7 @@ def set_layer_config(# 10. GGUF handlingif not gguf_name:dispatch_layer_config(layer_config)- return layer_config, has_qlayer_outside_block+ return layer_config, has_qlayer_outside_block, regex_config# embed + lm_head defaults for ggufif lm_head_name not in layer_config and not tie_word_embeddings:@@ -3019,7 +3022,7 @@ def set_layer_config(layer_config, _ = get_layer_config_by_gguf_format(layer_config, gguf_name.lower(), model, model_type)dispatch_layer_config(layer_config)- return layer_config, has_qlayer_outside_block+ return layer_config, has_qlayer_outside_block, regex_configdef check_diffusers_installed(): # pragma: no cover@@ -3055,3 +3058,93 @@ def is_diffusion_model(model_or_path: Union[str, object]) -> bool:return isinstance(model_or_path, pipeline_utils.DiffusionPipeline)else:return False+++def to_standard_regex(pattern: str) -> str:+ """+ Convert a user-specified string into a standardized regex for layer matching.++ Rules:+ - If the pattern already contains regex tokens ('.*', '^', '$', etc.),+ keep them as-is.+ - Otherwise, wrap the pattern with `.*` on both sides to allow substring matching.+ - Always ensure the returned regex is valid (compilable by re).++ Examples:+ >>> to_standard_regex("model.embed_tokens")+ '.*model\\.embed_tokens.*'+ >>> to_standard_regex("mlp.gate")+ '.*mlp\\.gate.*'+ >>> to_standard_regex("mlp.gate$")+ '.*mlp\\.gate$'+ >>> to_standard_regex("mlp.*gate")+ '.*mlp.*gate.*'+ """+ # Heuristic: if pattern contains regex meta characters, assume partial regex+ meta_chars = {".*", "^", "$", "|", "(", ")", "[", "]", "?", "+"}+ has_regex = any(tok in pattern for tok in meta_chars)+ if not has_regex:+ # Escape literal dots, etc., and wrap with .* for substring matching+ pattern = re.escape(pattern)+ regex = f".*{pattern}.*"+ else:+ # Only escape bare dots that are not already part of regex constructs+ # Avoid double escaping .* sequences+ tmp = []+ i = 0+ while i < len(pattern):+ if pattern[i] == ".":+ if i + 1 < len(pattern) and pattern[i + 1] == "*":+ tmp.append(".*") # keep regex token+ i += 2+ continue+ else:+ tmp.append("\\.") # escape bare dot+ else:+ tmp.append(pattern[i])+ i += 1+ regex = "".join(tmp)+ # If no anchors are provided, allow substring matching+ if not regex.startswith("^") and not regex.startswith(".*"):+ regex = ".*" + regex+ if not regex.endswith("$") and not regex.endswith(".*"):+ regex = regex + ".*"+ # Validate regex+ try:+ re.compile(regex)+ except re.error as e:+ raise ValueError(f"Invalid regex generated from pattern '{pattern}': {e}")+ return regex+++def matches_any_regex(layer_name: str, regex_config: Dict[str, dict]) -> bool:+ """+ Check whether `layer_name` matches any regex pattern key in `regex_config`.+ Args:+ layer_name (str): The layer name to test.+ regex_config (Dict[str, dict]): A mapping of regex patterns to configs.+ Returns:+ bool: True if any pattern matches `layer_name`, otherwise False.+ """+ if not regex_config:+ return False++ for pattern in regex_config:+ # Strip dynamic prefixes (e.g., "+:" or "-:")+ raw_pattern = pattern[2:] if pattern.startswith(("+:", "-:")) else pattern++ try:+ if re.search(raw_pattern, layer_name):+ return True+ except re.error as e:+ logger.warning("Skipping invalid regex pattern %r: %s", pattern, e)+ continue++ return False+++def json_serialize(obj: Any):+ """Convert non-JSON-serializable objects into JSON-friendly formats."""+ if isinstance(obj, torch.dtype):+ return str(obj).split(".")[-1] # e.g., torch.float16 -> "float16"+ raise TypeError(f"Object of type {type(obj).__name__} is not JSON serializable")diff --git a/test/test_cpu/test_mix_bits.py b/test/test_cpu/test_mix_bits.pynew file mode 100644index 000000000..2c73d42cd--- /dev/null+++ b/test/test_cpu/test_mix_bits.py@@ -0,0 +1,254 @@+import json+import os+import shutil+import sys+import unittest+from pathlib import Path++from parameterized import parameterized++sys.path.insert(0, "../..")+import torch+from transformers import AutoModelForCausalLM, AutoRoundConfig, AutoTokenizer++from auto_round import AutoRound+from auto_round.testing_utils import require_gptqmodel+++def _get_folder_size(path: str) -> float:+ """Return folder size in GB."""+ total_size = 0+ for dirpath, _, filenames in os.walk(path):+ for f in filenames:+ fp = os.path.join(dirpath, f)+ if os.path.isfile(fp):+ total_size += os.path.getsize(fp)+ return total_size / (1024**3) # convert to GB+++class LLMDataLoader:+ def __init__(self):+ self.batch_size = 1++ def __iter__(self):+ for i in range(2):+ yield torch.ones([1, 10], dtype=torch.long)+++class TestAutoRound(unittest.TestCase):+ @classmethod+ def setUpClass(self):+ self.model_name = "/tf_dataset/auto_round/models/facebook/opt-125m"+ self.save_dir = ".saved/"+ self.model = AutoModelForCausalLM.from_pretrained(self.model_name, torch_dtype="auto", trust_remote_code=True)+ self.tokenizer = AutoTokenizer.from_pretrained(self.model_name, trust_remote_code=True)+ self.llm_dataloader = LLMDataLoader()++ @classmethod+ def tearDownClass(self):+ shutil.rmtree("./saved", ignore_errors=True)+ shutil.rmtree("runs", ignore_errors=True)++ @require_gptqmodel+ def test_mixed_gptqmodel(self):+ layer_config = {+ "k_proj": {"bits": 8}, # part name+ "lm_head": {"bits": 4}, # set lm_head quant+ "fc1": {"bits": 16},+ "model.decoder.layers.0.self_attn.v_proj": {"bits": 16},+ "model.decoder.layers.0.self_attn.q_proj": {"bits": 8}, # full name+ }+ autoround = AutoRound(+ model=self.model_name,+ scheme="W4A16",+ iters=2,+ seqlen=2,+ layer_config=layer_config,+ dataset=self.llm_dataloader,+ )+ quantized_model_path = self.save_dir+ autoround.quantize_and_save(output_dir=quantized_model_path, format="auto_gptq")+ # test original GPTQModel inference+ from gptqmodel import GPTQModel++ model = GPTQModel.load(quantized_model_path)+ assert model.model.model.decoder.layers[0].self_attn.k_proj.bits == 8+ assert model.model.model.decoder.layers[0].self_attn.q_proj.bits == 8+ assert model.model.model.decoder.layers[1].self_attn.v_proj.bits == 4+ result = model.generate("Uncovering deep insights begins with")[0] # tokens+ assert "!!!" not in model.tokenizer.decode(result) # string output+ shutil.rmtree(quantized_model_path, ignore_errors=True)++ def test_mixed_gptqmodel_convert_to_ar(self):+ layer_config = {+ "k_proj": {"bits": 8}, # part name+ "lm_head": {"bits": 4}, # set lm_head quant+ "fc1": {"bits": 16},+ "model.decoder.layers.0.self_attn.v_proj": {"bits": 16},+ "model.decoder.layers.0.self_attn.q_proj": {"bits": 8}, # full name+ }+ autoround = AutoRound(+ model=self.model_name,+ scheme="W4A16",+ iters=2,+ seqlen=2,+ layer_config=layer_config,+ dataset=self.llm_dataloader,+ )+ quantized_model_path = self.save_dir+ autoround.quantize_and_save(output_dir=quantized_model_path, format="auto_gptq")+ quantization_config = AutoRoundConfig()+ model = AutoModelForCausalLM.from_pretrained(+ quantized_model_path, device_map="cpu", quantization_config=quantization_config+ )+ tokenizer = AutoTokenizer.from_pretrained(quantized_model_path)+ text = "There is a girl who likes adventure,"+ inputs = tokenizer(text, return_tensors="pt").to(model.device)+ res = tokenizer.decode(model.generate(**inputs, max_new_tokens=5)[0])+ print(res)+ shutil.rmtree(quantized_model_path, ignore_errors=True)++ def test_mixed_autoround_format(self):+ layer_config = {+ "k_proj": {"bits": 8},+ "q_proj": {"bits": 3},+ "lm_head": {"bits": 16},+ "fc1": {"bits": 16},+ }+ autoround = AutoRound(+ model=self.model_name,+ scheme="W4A16",+ iters=2,+ seqlen=2,+ dataset=self.llm_dataloader,+ layer_config=layer_config,+ )+ quantized_model_path = "./saved"+ compressed_model = autoround.quantize_and_save(output_dir=quantized_model_path, format="auto_round")+ model = AutoModelForCausalLM.from_pretrained(quantized_model_path, device_map="cpu")+ assert model.model.decoder.layers[0].self_attn.k_proj.bits == 8+ assert model.model.decoder.layers[0].self_attn.q_proj.bits == 3+ tokenizer = AutoTokenizer.from_pretrained(quantized_model_path)+ text = "There is a girl who likes adventure,"+ inputs = tokenizer(text, return_tensors="pt").to(model.device)+ print(tokenizer.decode(model.generate(**inputs, max_new_tokens=50)[0]))+ shutil.rmtree(quantized_model_path, ignore_errors=True)++ def test_fallback_regex_for_awq_format(self):+ layer_config = {+ "lm_head": {"bits": 16},+ "fc1": {"bits": 16},+ }+ autoround = AutoRound(+ model=self.model_name,+ scheme="W4A16",+ iters=2,+ seqlen=2,+ dataset=self.llm_dataloader,+ layer_config=layer_config,+ )+ quantized_model_path = "./saved"+ autoround.quantize_and_save(output_dir=quantized_model_path, format="auto_awq")+ quantization_config = AutoRoundConfig()+ model = AutoModelForCausalLM.from_pretrained(+ quantized_model_path, device_map="cpu", quantization_config=quantization_config+ )+ tokenizer = AutoTokenizer.from_pretrained(quantized_model_path)+ text = "There is a girl who likes adventure,"+ inputs = tokenizer(text, return_tensors="pt").to(model.device)+ print(tokenizer.decode(model.generate(**inputs, max_new_tokens=50)[0]))+ shutil.rmtree(quantized_model_path, ignore_errors=True)++ def test_mixed_ar_format_part_name_hf_loading(self):+ layer_config = {+ "k_proj": {"bits": 8}, # part name+ "lm_head": {"bits": 16}, # full name+ ".*fc1.*": {"bits": 16}, # standard regex+ }+ autoround = AutoRound(+ model=self.model_name,+ scheme="W4A16",+ iters=2,+ seqlen=2,+ dataset=self.llm_dataloader,+ layer_config=layer_config,+ )+ quantized_model_path = "./saved"+ autoround.quantize()+ autoround.save_quantized(output_dir=quantized_model_path, format="auto_round")+ # remove old extra_config(which contains full name layer configs), only test regex config loading+ new_extra_config = {+ ".*fc1.*": { # standard regex+ "act_bits": 16,+ "act_data_type": "float",+ "act_dynamic": True,+ "act_group_size": 128,+ "act_sym": True,+ "bits": 16,+ "data_type": "int",+ "group_size": 128,+ "sym": True,+ },+ "k_proj": { # part name+ "act_bits": 16,+ "act_data_type": "float",+ "act_dynamic": True,+ "act_group_size": 128,+ "act_sym": True,+ "bits": 8,+ "data_type": "int",+ "group_size": 128,+ "sym": True,+ },+ }+ config_file = Path(quantized_model_path) / "config.json"+ with open(config_file, "r", encoding="utf-8") as f:+ config = json.load(f)+ quant_config = config.get("quantization_config", {})+ old_extra_config = quant_config.get("extra_config", {})+ quant_config["extra_config"] = new_extra_config+ config["quantization_config"] = quant_config+ with open(config_file, "w", encoding="utf-8") as f:+ json.dump(config, f, indent=2, ensure_ascii=False)++ model = AutoModelForCausalLM.from_pretrained(quantized_model_path, device_map="cpu")+ assert model.model.decoder.layers[0].self_attn.k_proj.bits == 8+ assert model.model.decoder.layers[0].self_attn.q_proj.bits == 4+ tokenizer = AutoTokenizer.from_pretrained(quantized_model_path)+ text = "There is a girl who likes adventure,"+ inputs = tokenizer(text, return_tensors="pt").to(model.device)+ print(tokenizer.decode(model.generate(**inputs, max_new_tokens=50)[0]))+ shutil.rmtree(quantized_model_path, ignore_errors=True)++ def test_mixed_MXFP_autoround_format_loading(self):+ layer_config = {+ "k_proj": {"bits": 8, "act_bits": 8},+ "lm_head": {"bits": 16, "act_bits": 16},+ "fc1": {"bits": 8, "act_bits": 8},+ }+ autoround = AutoRound(+ self.model_name,+ scheme="MXFP4",+ iters=2,+ seqlen=2,+ dataset=self.llm_dataloader,+ layer_config=layer_config,+ )+ quantized_model_path = self.save_dir+ autoround.quantize_and_save(output_dir=quantized_model_path, inplace=False, format="auto_round")+ model = AutoModelForCausalLM.from_pretrained(+ quantized_model_path,+ torch_dtype="auto",+ device_map="cpu",+ )+ tokenizer = AutoTokenizer.from_pretrained(quantized_model_path)+ from auto_round.eval.evaluation import simple_evaluate_user_model++ result = simple_evaluate_user_model(model, tokenizer, batch_size=16, tasks="lambada_openai", limit=10)+ print(result["results"]["lambada_openai"]["acc,none"])+ self.assertGreater(result["results"]["lambada_openai"]["acc,none"], 0.14)+ shutil.rmtree(quantized_model_path, ignore_errors=True)+++if __name__ == "__main__":+ unittest.main()diff --git a/test/test_cuda/test_mix_bits.py b/test/test_cuda/test_mix_bits.pynew file mode 100644index 000000000..4f7d39d8c--- /dev/null+++ b/test/test_cuda/test_mix_bits.py@@ -0,0 +1,329 @@+import json+import os+import shutil+import sys+import unittest++from parameterized import parameterized++sys.path.insert(0, "../..")+from pathlib import Path++import torch+from transformers import AutoModelForCausalLM, AutoRoundConfig, AutoTokenizer++from auto_round import AutoRound+from auto_round.testing_utils import (+ require_awq,+ require_gptqmodel,+ require_package_version_ut,+)+++class LLMDataLoader:+ def __init__(self):+ self.batch_size = 1++ def __iter__(self):+ for i in range(2):+ yield torch.ones([1, 10], dtype=torch.long)+++class TestAutoRound(unittest.TestCase):+ @classmethod+ def setUpClass(self):+ self.model_name = "/models/opt-125m"+ self.save_dir = "./saved"+ self.model = AutoModelForCausalLM.from_pretrained(self.model_name, torch_dtype="auto", trust_remote_code=True)+ self.tokenizer = AutoTokenizer.from_pretrained(self.model_name, trust_remote_code=True)+ self.llm_dataloader = LLMDataLoader()++ @classmethod+ def tearDownClass(self):+ shutil.rmtree("./saved", ignore_errors=True)+ shutil.rmtree("runs", ignore_errors=True)++ @require_gptqmodel+ def test_mixed_gptqmodel(self):+ scheme = "W4A16"+ layer_config = {+ "k_proj": {"bits": 8}, # part name+ "lm_head": {"bits": 4}, # set lm_head quant+ "fc1": {"bits": 16},+ "model.decoder.layers.0.self_attn.v_proj": {"bits": 16},+ "model.decoder.layers.0.self_attn.q_proj": {"bits": 8}, # full name+ }+ autoround = AutoRound(+ model=self.model_name,+ scheme=scheme,+ iters=2,+ seqlen=2,+ layer_config=layer_config,+ dataset=self.llm_dataloader,+ )+ quantized_model_path = self.save_dir+ autoround.quantize_and_save(output_dir=quantized_model_path, format="auto_gptq")+ from gptqmodel import GPTQModel++ model = GPTQModel.load(quantized_model_path)+ assert model.model.model.decoder.layers[0].self_attn.k_proj.bits == 8+ assert model.model.model.decoder.layers[0].self_attn.q_proj.bits == 8+ assert model.model.model.decoder.layers[1].self_attn.v_proj.bits == 4+ res = model.tokenizer.decode(model.generate("Uncovering deep insights begins with")[0])+ print(res)+ shutil.rmtree(quantized_model_path, ignore_errors=True)++ def test_mixed_gptqmodel_convert_to_ar(self):+ layer_config = {+ "k_proj": {"bits": 8}, # part name+ "lm_head": {"bits": 4}, # set lm_head quant+ "model.decoder.layers.0.self_attn.v_proj": {"bits": 16},+ "model.decoder.layers.0.self_attn.q_proj": {"bits": 8}, # full name+ }+ autoround = AutoRound(+ model=self.model_name,+ scheme="W4A16",+ iters=2,+ seqlen=2,+ layer_config=layer_config,+ dataset=self.llm_dataloader,+ )+ quantized_model_path = self.save_dir+ autoround.quantize_and_save(output_dir=quantized_model_path, format="auto_gptq")+ quantization_config = AutoRoundConfig()+ model = AutoModelForCausalLM.from_pretrained(+ quantized_model_path, device_map="auto", quantization_config=quantization_config+ )+ tokenizer = AutoTokenizer.from_pretrained(quantized_model_path)+ text = "There is a girl who likes adventure,"+ inputs = tokenizer(text, return_tensors="pt").to(model.device)+ res = tokenizer.decode(model.generate(**inputs, max_new_tokens=5)[0])+ print(res)+ shutil.rmtree(quantized_model_path, ignore_errors=True)++ def test_mixed_autoround_format(self):+ layer_config = {+ "k_proj": {"bits": 8},+ "q_proj": {"bits": 3},+ "lm_head": {"bits": 16},+ "fc1": {"bits": 16},+ }+ autoround = AutoRound(+ model=self.model_name,+ scheme="W4A16",+ iters=2,+ seqlen=2,+ dataset=self.llm_dataloader,+ layer_config=layer_config,+ )+ quantized_model_path = "self.save_dir"+ autoround.quantize_and_save(output_dir=quantized_model_path, format="auto_round")+ model = AutoModelForCausalLM.from_pretrained(quantized_model_path, device_map="auto")+ assert model.model.decoder.layers[0].self_attn.k_proj.bits == 8+ assert model.model.decoder.layers[0].self_attn.q_proj.bits == 3+ tokenizer = AutoTokenizer.from_pretrained(quantized_model_path)+ text = "There is a girl who likes adventure,"+ inputs = tokenizer(text, return_tensors="pt").to(model.device)+ print(tokenizer.decode(model.generate(**inputs, max_new_tokens=50)[0]))+ shutil.rmtree(quantized_model_path, ignore_errors=True)++ @require_awq+ @require_package_version_ut("transformers", "<4.57.0")+ def test_fallback_regex_for_awq_format(self):+ model_name = "facebook/opt-125m"+ layer_config = {+ "lm_head": {"bits": 16},+ "fc1": {"bits": 16},+ }+ autoround = AutoRound(+ model=model_name,+ scheme="W4A16",+ iters=2,+ seqlen=2,+ dataset=self.llm_dataloader,+ layer_config=layer_config,+ )+ quantized_model_path = "self.save_dir"+ autoround.quantize_and_save(output_dir=quantized_model_path, format="auto_awq")+ quantization_config = AutoRoundConfig()+ model = AutoModelForCausalLM.from_pretrained(+ quantized_model_path, device_map="auto", quantization_config=quantization_config+ )+ tokenizer = AutoTokenizer.from_pretrained(quantized_model_path)+ text = "There is a girl who likes adventure,"+ inputs = tokenizer(text, return_tensors="pt").to(model.device)+ print(tokenizer.decode(model.generate(**inputs, max_new_tokens=50)[0]))+ shutil.rmtree(quantized_model_path, ignore_errors=True)++ def test_mixed_ar_format_part_name_hf_loading(self):+ layer_config = {+ "k_proj": {"bits": 8}, # part name+ "lm_head": {"bits": 16}, # full name+ ".*fc1.*": {"bits": 16}, # standard regex+ }+ autoround = AutoRound(+ model=self.model_name,+ scheme="W4A16",+ iters=2,+ seqlen=2,+ dataset=self.llm_dataloader,+ layer_config=layer_config,+ )+ quantized_model_path = "self.save_dir"+ autoround.quantize()+ autoround.save_quantized(output_dir=quantized_model_path, format="auto_round")+ # remove old extra_config(which contains full name layer configs), only test regex config loading+ new_extra_config = {+ ".*fc1.*": { # standard regex+ "act_bits": 16,+ "act_data_type": "float",+ "act_dynamic": True,+ "act_group_size": 128,+ "act_sym": True,+ "bits": 16,+ "data_type": "int",+ "group_size": 128,+ "sym": True,+ },+ "k_proj": { # part name+ "act_bits": 16,+ "act_data_type": "float",+ "act_dynamic": True,+ "act_group_size": 128,+ "act_sym": True,+ "bits": 8,+ "data_type": "int",+ "group_size": 128,+ "sym": True,+ },+ }+ config_file = Path(quantized_model_path) / "config.json"+ with open(config_file, "r", encoding="utf-8") as f:+ config = json.load(f)+ quant_config = config.get("quantization_config", {})+ old_extra_config = quant_config.get("extra_config", {})+ quant_config["extra_config"] = new_extra_config+ config["quantization_config"] = quant_config+ with open(config_file, "w", encoding="utf-8") as f:+ json.dump(config, f, indent=2, ensure_ascii=False)++ model = AutoModelForCausalLM.from_pretrained(quantized_model_path, device_map="auto")+ assert model.model.decoder.layers[0].self_attn.k_proj.bits == 8+ assert model.model.decoder.layers[0].self_attn.q_proj.bits == 4+ tokenizer = AutoTokenizer.from_pretrained(quantized_model_path)+ text = "There is a girl who likes adventure,"+ inputs = tokenizer(text, return_tensors="pt").to(model.device)+ print(tokenizer.decode(model.generate(**inputs, max_new_tokens=50)[0]))+ shutil.rmtree(quantized_model_path, ignore_errors=True)++ def test_mixed_MXFP_autoround_format_loading(self):+ layer_config = {+ "k_proj": {"bits": 8, "act_bits": 8},+ "lm_head": {"bits": 16, "act_bits": 16},+ "fc1": {"bits": 8, "act_bits": 8},+ }+ autoround = AutoRound(+ self.model_name,+ scheme="MXFP4",+ iters=2,+ seqlen=2,+ dataset=self.llm_dataloader,+ layer_config=layer_config,+ )+ quantized_model_path = self.save_dir+ autoround.quantize_and_save(output_dir=quantized_model_path, inplace=False, format="auto_round")+ model = AutoModelForCausalLM.from_pretrained(+ quantized_model_path,+ torch_dtype="auto",+ device_map="auto",+ )+ tokenizer = AutoTokenizer.from_pretrained(quantized_model_path)+ from auto_round.eval.evaluation import simple_evaluate_user_model++ result = simple_evaluate_user_model(model, tokenizer, batch_size=16, tasks="lambada_openai")+ print(result["results"]["lambada_openai"]["acc,none"])+ self.assertGreater(result["results"]["lambada_openai"]["acc,none"], 0.32)+ shutil.rmtree(quantized_model_path, ignore_errors=True)++ def test_mixed_autoround_format_vllm(self):+ layer_config = {+ "self_attn": {"bits": 8},+ "lm_head": {"bits": 16},+ }+ autoround = AutoRound(+ self.model,+ self.tokenizer,+ scheme="W4A16",+ iters=2,+ seqlen=2,+ dataset=self.llm_dataloader,+ layer_config=layer_config,+ )+ autoround.quantize()+ quantized_model_path = self.save_dir+ autoround.save_quantized(output_dir=quantized_model_path, inplace=False, format="auto_round")++ from vllm import LLM, SamplingParams++ # Sample prompts.+ prompts = [+ "The capital of France is",+ "The future of AI is",+ ]+ # Create a sampling params object.+ sampling_params = SamplingParams(temperature=0.8, top_p=0.95)+ # Create an LLM.+ QUANTIZATION = "auto-round" # quantized_model_path+ llm = LLM(model=quantized_model_path, quantization=QUANTIZATION, trust_remote_code=True, tensor_parallel_size=1)+ outputs = llm.generate(prompts, sampling_params)+ # Print the outputs.+ for output in outputs:+ prompt = output.prompt+ generated_text = output.outputs[0].text+ # if "France" in prompt:+ assert "!!!" not in generated_text+ print(f"{prompt}: {generated_text}")+ shutil.rmtree(quantized_model_path, ignore_errors=True)++ def test_mixed_llmcompressor_format_vllm(self):+ layer_config = {+ "self_attn": {"bits": 16, "act_bits": 16},+ "lm_head": {"bits": 16, "act_bits": 16},+ "fc1": {"bits": 16, "act_bits": 16},+ }+ autoround = AutoRound(+ self.model_name,+ scheme="NVFP4",+ iters=2,+ seqlen=2,+ dataset=self.llm_dataloader,+ layer_config=layer_config,+ )+ quantized_model_path = self.save_dir+ compressed, _ = autoround.quantize_and_save(+ output_dir=quantized_model_path, inplace=False, format="llm_compressor"+ )+ from vllm import LLM, SamplingParams++ # Sample prompts.+ prompts = [+ "The capital of France is",+ "The future of AI is",+ ]+ # Create a sampling params object.+ sampling_params = SamplingParams(temperature=0.8, top_p=0.95)+ # Create an LLM.+ QUANTIZATION = "auto-round" # quantized_model_path+ llm = LLM(model=quantized_model_path, trust_remote_code=True, tensor_parallel_size=1)+ outputs = llm.generate(prompts, sampling_params)+ # Print the outputs.+ for output in outputs:+ prompt = output.prompt+ generated_text = output.outputs[0].text+ print(f"{prompt}: {generated_text}")+ assert "!!!" not in generated_text+ shutil.rmtree(quantized_model_path, ignore_errors=True)+++if __name__ == "__main__":+ unittest.main()