Skip to content

PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度

enable regex quantization config saving for mixed bits (#825)

Weiwei873114a0ff5e
diff --git a/auto_round/compressors/base.py b/auto_round/compressors/base.py
index d948de65b..f1c6c5bd4 100644
--- a/auto_round/compressors/base.py
+++ b/auto_round/compressors/base.py
@@ -439,7 +439,7 @@ class BaseCompressor(object):
"Please save the model using the `fake` format for now."
)
- layer_config, self.has_qlayer_outside_block = set_layer_config(
+ layer_config, self.has_qlayer_outside_block, self.regex_config = set_layer_config(
self.model,
self.layer_config,
self.scheme,
@@ -1653,7 +1653,7 @@ class BaseCompressor(object):
enable_gguf_official_mixed = True
else:
enable_gguf_official_mixed = False
- self.layer_config, self.has_qlayer_outside_block = set_layer_config(
+ self.layer_config, self.has_qlayer_outside_block, self.regex_config = set_layer_config(
self.model,
self.layer_config,
self.scheme,
@@ -2937,6 +2937,8 @@ class BaseCompressor(object):
"Support for exporting activation quantization is limited. "
"Please ensure that your configuration is supported."
)
+ # if format == "llm_compressor" and (is_nv_fp(self.data_type) or is_mx_fp(self.data_type)):
+ # format = format.replace("llm_compressor", f"llm_compressor:{self.data_type}")
if format == "llm_compressor" and (is_nv_fp(self.data_type) or is_mx_fp(self.data_type)):
format = format.replace("llm_compressor", f"llm_compressor:{self.data_type}")
if format == "llm_compressor" and is_static_wfp8afp8(self):
@@ -2985,6 +2987,7 @@ class BaseCompressor(object):
"act_data_type",
"super_bits",
"super_group_size",
+ "regex_config",
]
if isinstance(self.dataset, str):
serialization_keys.append("dataset")
diff --git a/auto_round/export/export_to_autogptq/export.py b/auto_round/export/export_to_autogptq/export.py
index 7e8497441..5c795bac9 100644
--- a/auto_round/export/export_to_autogptq/export.py
+++ b/auto_round/export/export_to_autogptq/export.py
@@ -17,6 +17,8 @@ import inspect
import json
import os
from concurrent.futures import ThreadPoolExecutor
+from dataclasses import fields
+from typing import Any, Dict
import threadpoolctl as tctl
@@ -48,16 +50,28 @@ from tqdm import tqdm
import auto_round.export.export_to_autogptq.qlinear_triton
from auto_round.export.utils import save_model
+from auto_round.schemes import QuantizationScheme
+
+GPTQ_REQUIRED_CONFIG_KEYS = (
+ "bits",
+ "group_size",
+ "sym",
+)
+
from auto_round.logger import logger
from auto_round.utils import (
SUPPORTED_LAYER_TYPES,
+ check_start_with_block_name,
check_to_quantized,
copy_python_files_from_model_cache,
filter_quantization_config,
get_autogptq_packing_qlinear,
get_block_names,
get_module,
+ json_serialize,
+ matches_any_regex,
set_module,
+ to_standard_regex,
)
BLOCK_PATTERNS = [ ## copy from transformers optimum
@@ -66,6 +80,54 @@ BLOCK_PATTERNS = [ ## copy from transformers optimum
"gpt_neox.layers",
"model.layers",
]
+from auto_round.export.export_to_autoround.utils import check_neq_config
+
+
+def convert_to_autogptq_dynamic(regex_config: Dict[str, Dict[str, Any]]) -> Dict[str, Dict[str, Any]]:
+ """
+ Convert AutoRound-style regex_config into AutoGPTQ-style QuantizerConfig.dynamic.
+
+ Rules:
+ - bits < 16 -> quantize -> positive match `+:regex`
+ - bits == 16 -> skip quantize -> negative match `-:regex`
+ """
+ converted = {}
+ for name, cfg in regex_config.items():
+ bits = cfg.get("bits")
+ regex = to_standard_regex(name)
+
+ if bits is None:
+ continue # ignore invalid entries
+ elif bits < 16:
+ converted[f"+:{regex}"] = {"bits": bits}
+ for key in GPTQ_REQUIRED_CONFIG_KEYS: # only save keys gptq supported
+ converted[f"+:{regex}"][key] = regex_config[name][key]
+ else:
+ # skip quantization
+ converted[f"-:{regex}"] = {}
+ return converted
+
+
+def convert_from_autogptq_dynamic(dynamic_config: dict) -> dict:
+ """
+ Convert AutoGPTQ-style QuantizerConfig.dynamic into AutoRound-style extra_config.
+
+ Rules:
+ - '+:regex' => quantize => keep bits and other quantization keys
+ - '-:regex' => skip quantize => set bits to 16 (FP16 passthrough)
+ """
+ converted = {}
+ for name, cfg in dynamic_config.items():
+ # Strip the +: or -:
+ if name.startswith("+:"):
+ regex = name[2:]
+ # keep all config fields (bits, group_size, sym, etc.)
+ converted[regex] = dict(cfg)
+ elif name.startswith("-:"):
+ regex = name[2:]
+ # mark skipped layers with bits=16
+ converted[regex] = {"bits": 16, "act_bits": 16}
+ return converted
def pack_layer(name, model, backend, device=None):
@@ -132,58 +194,93 @@ def pack_layer(name, model, backend, device=None):
def save_quantized_as_autogptq(output_dir, inplace=True, backend="auto_gptq:exllamav2", **kwargs):
"""Export the model to autogptq format to easily leverage cuda kernel."""
+ # --- 1️⃣ Extract inputs & configs ---
model = kwargs["model"]
- safe_serialization = True if "safe_serialization" not in kwargs.keys() else kwargs["safe_serialization"]
- quant_block_list = kwargs.get("quant_block_list", get_block_names(model))
- tokenizer = kwargs.get("tokenizer", None)
- processor = kwargs.get("processor", None)
- device = kwargs.get("device", None)
- image_processor = kwargs.get("image_processor", None)
- if output_dir is not None and os.path.exists(output_dir):
- logger.warning(f"{output_dir} already exists, this may cause model conflict")
- if output_dir is not None and tokenizer is not None and hasattr(tokenizer, "save_pretrained"):
- tokenizer.save_pretrained(output_dir)
- if output_dir is not None and processor is not None:
- processor.save_pretrained(output_dir)
- if output_dir is not None and image_processor is not None:
- image_processor.save_pretrained(output_dir)
- ##check module quantized in block, this may have bug for mixed precision quantization
quantization_config = kwargs["serialization_dict"]
+ layer_config = kwargs["layer_config"]
+ quant_block_list = kwargs.get("quant_block_list", get_block_names(model))
+ tokenizer = kwargs.get("tokenizer")
+ processor = kwargs.get("processor")
+ image_processor = kwargs.get("image_processor")
+ device = kwargs.get("device")
+ safe_serialization = kwargs.get("safe_serialization", True)
+
+ # --- Save metadata (tokenizer, processor, etc.) ---
+ if output_dir:
+ if os.path.exists(output_dir):
+ logger.warning(f"{output_dir} already exists, may cause overwrite conflicts.")
+ for comp in (tokenizer, processor, image_processor):
+ if comp is not None and hasattr(comp, "save_pretrained"):
+ comp.save_pretrained(output_dir)
+
+ # --- Handle quantization structure ---
all_blocks = quant_block_list
- flattened_list = [item for sublist in all_blocks for item in sublist]
- common_prefix = os.path.commonprefix(flattened_list).rstrip(".")
- if common_prefix not in BLOCK_PATTERNS:
- logger.error("auto-gptq format may not support loading this quantized model")
+ flattened = [x for sub in all_blocks for x in sub]
+ common_prefix = os.path.commonprefix(flattened).rstrip(".")
+
+ if "BLOCK_PATTERNS" in kwargs and common_prefix not in kwargs["BLOCK_PATTERNS"]:
+ logger.error(f"Unsupported block prefix '{common_prefix}' for AutoGPTQ format.")
quantization_config["block_name_to_quantize"] = common_prefix
quantization_config.pop("to_quant_block_names", None)
- ## as layers maybe already packed, we need to check in layer_config
- layer_config = kwargs["layer_config"]
+ # --- Build per-layer dynamic overrides ---
+ regex_config = quantization_config.pop("regex_config", {})
+ block_name_to_quantize = quantization_config.get("block_name_to_quantize")
+ extra_config = {}
+ lm_head_quantized = False
+ scheme_keys = [f.name for f in fields(QuantizationScheme)]
+ for layer_name, cfg in layer_config.items():
+ bits = cfg.get("bits", 16)
+ in_blocks = cfg.get("in_blocks", False)
+ # Handle non-block layers (e.g., LM head)
+ if not in_blocks and bits <= 8:
+ lm_head_quantized = True
+ extra_config[layer_name] = {k: cfg[k] for k in GPTQ_REQUIRED_CONFIG_KEYS}
+ continue
+ # Handle block layers
+ if in_blocks or (block_name_to_quantize and check_start_with_block_name(layer_name, block_name_to_quantize)):
+ neq_keys = check_neq_config(cfg, **{k: quantization_config[k] for k in scheme_keys})
+ if neq_keys:
+ if matches_any_regex(layer_name, regex_config):
+ continue
+ extra_config[layer_name] = {k: cfg[k] for k in GPTQ_REQUIRED_CONFIG_KEYS}
+
+ # --- Merge regex_config + extra_config into GPTQ dynamic config ---
+ dynamic = {}
+ if regex_config:
+ dynamic.update(convert_to_autogptq_dynamic(regex_config))
+ if extra_config:
+ dynamic.update(convert_to_autogptq_dynamic(extra_config))
+ if dynamic:
+ quantization_config["dynamic"] = dynamic
+
+ # --- Block-wise quantization verification ---
for n, m in model.named_modules():
m.tmp_name = n
all_to_quantized = True
modules_in_block_to_quantize = []
- for block_names in all_blocks:
- first_block = get_module(model, block_names[0])
- for n, m in first_block.named_modules():
- if m.tmp_name not in layer_config.keys():
- continue
- if not check_to_quantized(layer_config[m.tmp_name]):
- all_to_quantized = False
- else:
- modules_in_block_to_quantize.append(n)
- modules_in_block_to_quantize = [modules_in_block_to_quantize]
+ if not dynamic: # Only uniform precision
+ for block_names in all_blocks:
+ first_block = get_module(model, block_names[0])
+ for n, m in first_block.named_modules():
+ if m.tmp_name not in layer_config:
+ continue
+ if not check_to_quantized(layer_config[m.tmp_name]):
+ all_to_quantized = False
+ else:
+ modules_in_block_to_quantize.append(n)
+ modules_in_block_to_quantize = [modules_in_block_to_quantize]
+
if all_to_quantized:
modules_in_block_to_quantize = None
- for n, m in model.named_modules():
+ for _, m in model.named_modules():
delattr(m, "tmp_name")
if not inplace:
model = copy.deepcopy(model.to("cpu"))
- layer_config = kwargs["layer_config"]
names = list(layer_config.keys())
max_workers = 1
if not torch.cuda.is_available() and not torch.xpu.is_available():
@@ -202,6 +299,7 @@ def save_quantized_as_autogptq(output_dir, inplace=True, backend="auto_gptq:exll
pass
if output_dir is None:
return model
+ quantization_config["lm_head"] = lm_head_quantized
quantization_config["provider"] = "auto-round"
quantization_config["quant_method"] = "gptq"
quantization_config.pop("dataset", None) ## pile-10k is not supported in gptq
diff --git a/auto_round/export/export_to_autoround/export.py b/auto_round/export/export_to_autoround/export.py
index b31b81a16..679257762 100644
--- a/auto_round/export/export_to_autoround/export.py
+++ b/auto_round/export/export_to_autoround/export.py
@@ -44,6 +44,7 @@ from auto_round.utils import (
is_nv_fp,
is_standard_fp,
set_module,
+ to_standard_regex,
)
@@ -340,8 +341,15 @@ def save_quantized_as_autoround(output_dir, inplace=True, backend="auto_round:ex
if cfg[key] is not None:
extra_config[layer_name][key] = cfg[key]
+ regex_config = quantization_config.pop("regex_config")
+ if regex_config is not None:
+ for name in regex_config.keys():
+ regex_name = to_standard_regex(name)
+ extra_config[regex_name] = {**{k: regex_config[name][k] for k in scheme_keys}}
+
if len(extra_config) > 0:
quantization_config["extra_config"] = extra_config
+
names = list(layer_config.keys())
max_workers = 1
if not torch.cuda.is_available() and not torch.xpu.is_available():
diff --git a/auto_round/export/export_to_autoround/export_to_nvfp_mxfp.py b/auto_round/export/export_to_autoround/export_to_nvfp_mxfp.py
index 9e3a73533..e344b51b3 100644
--- a/auto_round/export/export_to_autoround/export_to_nvfp_mxfp.py
+++ b/auto_round/export/export_to_autoround/export_to_nvfp_mxfp.py
@@ -41,6 +41,7 @@ from auto_round.utils import (
is_nv_fp,
set_amax_for_all_moe_layers,
set_module,
+ to_standard_regex,
)
from auto_round.wrapper import WrapperWALayer
@@ -211,6 +212,12 @@ def save_quantized_as_fp(output_dir, inplace=True, **kwargs):
if cfg[key] is not None:
extra_config[layer_name][key] = cfg[key]
+ regex_config = quantization_config.pop("regex_config")
+ if regex_config is not None:
+ for name in regex_config.keys():
+ regex_name = to_standard_regex(name)
+ extra_config[regex_name] = {**{k: regex_config[name][k] for k in scheme_keys}}
+
if len(extra_config) > 0:
quantization_config["extra_config"] = extra_config
names = list(layer_config.keys())
diff --git a/auto_round/export/export_to_awq/export.py b/auto_round/export/export_to_awq/export.py
index a1f2d4d26..2cf256c67 100644
--- a/auto_round/export/export_to_awq/export.py
+++ b/auto_round/export/export_to_awq/export.py
@@ -137,6 +137,7 @@ def save_quantized_as_autoawq(output_dir, inplace=True, **kwargs):
return model
quantization_config = kwargs["serialization_dict"]
+ regex_config = quantization_config.pop("regex_config", {}) # awq do not support mixed bits config saving
if output_dir is None:
return compressed_model
@@ -145,11 +146,16 @@ def save_quantized_as_autoawq(output_dir, inplace=True, **kwargs):
for key in layer_config.keys():
if not check_to_quantized(layer_config[key]) and not any(name in key for name in modules_to_not_convert):
modules_to_not_convert.append(key)
+ for key, cfg in regex_config.items():
+ bits = cfg.get("bits")
+ if bits > 8: # save fp_layer regexs
+ modules_to_not_convert.append(key)
+
quantization_config["provider"] = "auto-round"
quantization_config["quant_method"] = "awq"
quantization_config["zero_point"] = not quantization_config["sym"]
quantization_config["version"] = "gemm"
- quantization_config["modules_to_not_convert"] = modules_to_not_convert
+ quantization_config["modules_to_not_convert"] = list(dict.fromkeys(modules_to_not_convert))
##check module quantized in block, this may have bug for mixed precision quantization
filter_quantization_config(quantization_config)
if hasattr(compressed_model, "config"):
diff --git a/auto_round/export/export_to_llmcompressor/export_to_fp.py b/auto_round/export/export_to_llmcompressor/export_to_fp.py
index 633593075..f5c273856 100644
--- a/auto_round/export/export_to_llmcompressor/export_to_fp.py
+++ b/auto_round/export/export_to_llmcompressor/export_to_fp.py
@@ -25,6 +25,7 @@ import transformers
from tqdm import tqdm
from auto_round.export.export_to_autoround.qlinear_fp import QuantLinear
+from auto_round.export.export_to_llmcompressor.utils import generate_ignore_regex_list
from auto_round.export.utils import save_model
from auto_round.logger import logger
from auto_round.utils import (
@@ -114,9 +115,8 @@ def pack_layer(name, model, backend, device=None):
scale = layer.scale
global_scale = getattr(layer, "weight_global_scale", None)
input_global_scale = getattr(layer, "input_global_scale", None)
- # zero = layer.zp
+ # zero = layer.zp # no zeros to handle, as mxfp not support asym quantization
qlayer.pack(layer, scale, global_scale=global_scale, input_global_scale=input_global_scale, device=device)
- ## no zeros to handle, as mxfp not support asym quantization
qlayer.to(orig_device)
@@ -155,6 +155,9 @@ def save_quantized_as_fp(output_dir, inplace=True, **kwargs):
device = kwargs.get("device", None)
tokenizer = kwargs.get("tokenizer", None)
processor = kwargs.get("processor", None)
+ ar_quantization_config = kwargs["serialization_dict"]
+ regex_config = ar_quantization_config.pop("regex_config")
+ layer_config = kwargs["layer_config"]
extra_config = {}
if act_bits <= 8:
@@ -199,12 +202,7 @@ def save_quantized_as_fp(output_dir, inplace=True, **kwargs):
for _ in executor.map(wrapper, names):
pass
- # TODO fix the ignore re match issue, compile with fp8 & int8 config
- ignore = ["lm_head"]
- for layer_name in layer_config:
- if layer_config[layer_name]["bits"] > 8: ## find ignore layers
- ignore.append(layer_name)
- ignore = list(set(ignore))
+ ignore = generate_ignore_regex_list(regex_config=regex_config, layer_config=layer_config)
# get llm-compressor format config
check_compressed_tensors_supported()
diff --git a/auto_round/export/export_to_llmcompressor/utils.py b/auto_round/export/export_to_llmcompressor/utils.py
new file mode 100644
index 000000000..304a720bd
--- /dev/null
+++ b/auto_round/export/export_to_llmcompressor/utils.py
@@ -0,0 +1,51 @@
+# Copyright (c) 2025 Intel Corporation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from typing import Dict, List
+
+from auto_round.utils import matches_any_regex, to_standard_regex
+
+
+def generate_ignore_regex_list(regex_config: Dict[str, Dict], layer_config: Dict[str, Dict]) -> List[str]:
+ """
+ Generate ignore regex list for llm_compressor based on regex_config and layer_config.
+
+ Rules:
+ 1. Any layer in regex_config with bits >= 16 is ignored.
+ 2. Any layer in layer_config with bits >= 16 is ignored if not already included.
+ 3. Output regex patterns are normalized for llm_compressor ('re:...' style).
+
+ Args:
+ regex_config (Dict[str, Dict]): dynamic quantization config
+ layer_config (Dict[str, Dict]): layer-wise quantization config
+
+ Returns:
+ List[str]: List of regex patterns to ignore during quantization.
+ """
+ prefix = "re:"
+ ignore_regex: List[str] = []
+
+ # Step 1: Add regex_config keys with bits >= 16
+ for key, cfg in regex_config.items():
+ bits = cfg.get("bits")
+ if bits > 8:
+ ignore_regex.append(prefix + to_standard_regex(key))
+
+ # Step 2: Add all full named layer from layer_config if bits >= 16
+ for key, cfg in layer_config.items():
+ bits = cfg.get("bits")
+ if bits > 8:
+ ignore_regex.append(key)
+
+ return ignore_regex
diff --git a/auto_round/inference/convert_model.py b/auto_round/inference/convert_model.py
index 2c27c0374..a01623f73 100644
--- a/auto_round/inference/convert_model.py
+++ b/auto_round/inference/convert_model.py
@@ -28,6 +28,7 @@ from auto_round.inference.backend import (
get_layer_backend,
process_requirement,
)
+from auto_round.inference.utils import _expand_regex_config
from auto_round.logger import logger
from auto_round.schemes import QuantizationScheme
from auto_round.utils import (
@@ -258,21 +259,39 @@ def get_layer_config(model, quantization_config):
if not any([re.search(re.compile(n), layer_name) is not None for n in modules_in_block_to_quantize]):
extra_config[layer_name] = {"bits": 16} # Default to 16-bit for unquantized layers
- # Process AWQ format: exclude specified modules from quantization
+ # Expand GPTQ 'dynamic' config (regex-based)
+ dynamic_config = getattr(quantization_config, "dynamic", None)
+ from auto_round.export.export_to_autogptq.export import convert_from_autogptq_dynamic
+
+ if dynamic_config and isinstance(dynamic_config, dict):
+ extra_config = _expand_regex_config(
+ regex_config=convert_from_autogptq_dynamic(dynamic_config),
+ base_config=extra_config,
+ layer_names=layer_names,
+ model=model,
+ )
+
+ # AWQ format: exclude specified modules
extra_config = skip_not_convert_modules(model, quantization_config, layer_names, extra_config)
- # Merge and deduplicate layer names
+ # Expand auto_round regex configs (regex-based)
+ extra_config = _expand_regex_config(
+ regex_config=extra_config, base_config=extra_config, layer_names=layer_names, model=model
+ )
+
+ # Merge and deduplicate
layer_names = list(set(layer_names).union(extra_config.keys()))
- # Construct final layer configuration
+ # Build final layer configs
layer_configs = {}
quant_scheme_attrs = QuantizationScheme.get_attributes()
for layer_name in layer_names:
- layer_config = {}
- layer_extra_config = extra_config.get(layer_name, {})
- for scheme_attr in quant_scheme_attrs:
- layer_config[scheme_attr] = layer_extra_config.get(scheme_attr, getattr(default_quant_scheme, scheme_attr))
- layer_configs[layer_name] = QuantizationScheme.from_dict(layer_config)
+ layer_cfg_dict = {}
+ layer_extra = extra_config.get(layer_name, {})
+ for attr in quant_scheme_attrs:
+ layer_cfg_dict[attr] = layer_extra.get(attr, getattr(default_quant_scheme, attr))
+ layer_configs[layer_name] = QuantizationScheme.from_dict(layer_cfg_dict)
+
return layer_configs
diff --git a/auto_round/inference/utils.py b/auto_round/inference/utils.py
new file mode 100644
index 000000000..fdfa44161
--- /dev/null
+++ b/auto_round/inference/utils.py
@@ -0,0 +1,62 @@
+# Copyright (c) 2025 Intel Corporation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+import re
+
+from auto_round.utils import SUPPORTED_LAYER_TYPES
+
+
+def _expand_regex_config(regex_config, base_config, layer_names, model):
+ """
+ Expand regex-based layer configs to full layer names.
+
+ Args:
+ regex_config (dict): regex-based config (dynamic_config or part of extra_config)
+ base_config (dict): extra_config to write into
+ layer_names (list): known quantization layer names
+ model (nn.Module): target model
+
+ Returns:
+ dict: expanded base_config
+ """
+ if not regex_config:
+ return base_config
+
+ # Collect all supported layer names in model
+ all_supported_layer_names = [n for n, m in model.named_modules() if isinstance(m, SUPPORTED_LAYER_TYPES)]
+
+ # Identify which keys are regex patterns (not exact layer names)
+ regex_keys = [k for k in regex_config.keys() if k not in all_supported_layer_names]
+
+ for regex_key in regex_keys:
+ try:
+ pattern = re.compile(regex_key)
+ except re.error:
+ # invalid regex, skip silently
+ continue
+
+ # Prefer matches within layer_names first
+ matched_layers = [ln for ln in layer_names if re.search(pattern, ln)]
+ if not matched_layers:
+ matched_layers = [ln for ln in all_supported_layer_names if re.search(pattern, ln)]
+
+ if matched_layers:
+ cfg = regex_config[regex_key]
+ if cfg == {}:
+ continue
+ for ln in matched_layers:
+ # do not overwrite explicit layer config
+ if ln not in base_config:
+ base_config[ln] = cfg
+
+ return base_config
diff --git a/auto_round/utils.py b/auto_round/utils.py
index c73d6a210..84ecadd76 100644
--- a/auto_round/utils.py
+++ b/auto_round/utils.py
@@ -25,7 +25,7 @@ from dataclasses import asdict, fields
from enum import Enum
from functools import lru_cache
from pathlib import Path
-from typing import Any, Callable, Tuple, Union
+from typing import Any, Callable, Dict, List, Tuple, Union
import cpuinfo
import torch
@@ -2828,7 +2828,7 @@ def set_layer_config(
quant_lm_head: bool = False,
enable_gguf_official_mixed: bool = True,
is_mllm: bool = False,
-) -> tuple[dict, bool]:
+) -> tuple[dict, bool, dict]:
"""
Normalize, validate, and expand layer-specific quantization configs.
Returns (final_layer_config, has_quant_layer_outside_block)
@@ -2923,6 +2923,7 @@ def set_layer_config(
embedding_layer_names.append(n)
# 6. expand regex configs
+ regex_config = {}
for name in list(layer_config.keys()):
if name in all_supported_layer_names:
continue
@@ -2938,8 +2939,10 @@ def set_layer_config(
if not matched:
raise ValueError(f"Invalid '{name}' in layer_config, no match found.")
val = layer_config.pop(name)
+ regex_config[name] = val # keep regex config
for match in matched:
layer_config[match] = val
+ # regex_config = None if len(regex_config)==0 else regex_config
# 7. lm_head
lm_head_name = get_lm_head_name(model)
@@ -2996,7 +2999,7 @@ def set_layer_config(
# 10. GGUF handling
if not gguf_name:
dispatch_layer_config(layer_config)
- return layer_config, has_qlayer_outside_block
+ return layer_config, has_qlayer_outside_block, regex_config
# embed + lm_head defaults for gguf
if lm_head_name not in layer_config and not tie_word_embeddings:
@@ -3019,7 +3022,7 @@ def set_layer_config(
layer_config, _ = get_layer_config_by_gguf_format(layer_config, gguf_name.lower(), model, model_type)
dispatch_layer_config(layer_config)
- return layer_config, has_qlayer_outside_block
+ return layer_config, has_qlayer_outside_block, regex_config
def check_diffusers_installed(): # pragma: no cover
@@ -3055,3 +3058,93 @@ def is_diffusion_model(model_or_path: Union[str, object]) -> bool:
return isinstance(model_or_path, pipeline_utils.DiffusionPipeline)
else:
return False
+
+
+def to_standard_regex(pattern: str) -> str:
+ """
+ Convert a user-specified string into a standardized regex for layer matching.
+
+ Rules:
+ - If the pattern already contains regex tokens ('.*', '^', '$', etc.),
+ keep them as-is.
+ - Otherwise, wrap the pattern with `.*` on both sides to allow substring matching.
+ - Always ensure the returned regex is valid (compilable by re).
+
+ Examples:
+ >>> to_standard_regex("model.embed_tokens")
+ '.*model\\.embed_tokens.*'
+ >>> to_standard_regex("mlp.gate")
+ '.*mlp\\.gate.*'
+ >>> to_standard_regex("mlp.gate$")
+ '.*mlp\\.gate$'
+ >>> to_standard_regex("mlp.*gate")
+ '.*mlp.*gate.*'
+ """
+ # Heuristic: if pattern contains regex meta characters, assume partial regex
+ meta_chars = {".*", "^", "$", "|", "(", ")", "[", "]", "?", "+"}
+ has_regex = any(tok in pattern for tok in meta_chars)
+ if not has_regex:
+ # Escape literal dots, etc., and wrap with .* for substring matching
+ pattern = re.escape(pattern)
+ regex = f".*{pattern}.*"
+ else:
+ # Only escape bare dots that are not already part of regex constructs
+ # Avoid double escaping .* sequences
+ tmp = []
+ i = 0
+ while i < len(pattern):
+ if pattern[i] == ".":
+ if i + 1 < len(pattern) and pattern[i + 1] == "*":
+ tmp.append(".*") # keep regex token
+ i += 2
+ continue
+ else:
+ tmp.append("\\.") # escape bare dot
+ else:
+ tmp.append(pattern[i])
+ i += 1
+ regex = "".join(tmp)
+ # If no anchors are provided, allow substring matching
+ if not regex.startswith("^") and not regex.startswith(".*"):
+ regex = ".*" + regex
+ if not regex.endswith("$") and not regex.endswith(".*"):
+ regex = regex + ".*"
+ # Validate regex
+ try:
+ re.compile(regex)
+ except re.error as e:
+ raise ValueError(f"Invalid regex generated from pattern '{pattern}': {e}")
+ return regex
+
+
+def matches_any_regex(layer_name: str, regex_config: Dict[str, dict]) -> bool:
+ """
+ Check whether `layer_name` matches any regex pattern key in `regex_config`.
+ Args:
+ layer_name (str): The layer name to test.
+ regex_config (Dict[str, dict]): A mapping of regex patterns to configs.
+ Returns:
+ bool: True if any pattern matches `layer_name`, otherwise False.
+ """
+ if not regex_config:
+ return False
+
+ for pattern in regex_config:
+ # Strip dynamic prefixes (e.g., "+:" or "-:")
+ raw_pattern = pattern[2:] if pattern.startswith(("+:", "-:")) else pattern
+
+ try:
+ if re.search(raw_pattern, layer_name):
+ return True
+ except re.error as e:
+ logger.warning("Skipping invalid regex pattern %r: %s", pattern, e)
+ continue
+
+ return False
+
+
+def json_serialize(obj: Any):
+ """Convert non-JSON-serializable objects into JSON-friendly formats."""
+ if isinstance(obj, torch.dtype):
+ return str(obj).split(".")[-1] # e.g., torch.float16 -> "float16"
+ raise TypeError(f"Object of type {type(obj).__name__} is not JSON serializable")
diff --git a/test/test_cpu/test_mix_bits.py b/test/test_cpu/test_mix_bits.py
new file mode 100644
index 000000000..2c73d42cd
--- /dev/null
+++ b/test/test_cpu/test_mix_bits.py
@@ -0,0 +1,254 @@
+import json
+import os
+import shutil
+import sys
+import unittest
+from pathlib import Path
+
+from parameterized import parameterized
+
+sys.path.insert(0, "../..")
+import torch
+from transformers import AutoModelForCausalLM, AutoRoundConfig, AutoTokenizer
+
+from auto_round import AutoRound
+from auto_round.testing_utils import require_gptqmodel
+
+
+def _get_folder_size(path: str) -> float:
+ """Return folder size in GB."""
+ total_size = 0
+ for dirpath, _, filenames in os.walk(path):
+ for f in filenames:
+ fp = os.path.join(dirpath, f)
+ if os.path.isfile(fp):
+ total_size += os.path.getsize(fp)
+ return total_size / (1024**3) # convert to GB
+
+
+class LLMDataLoader:
+ def __init__(self):
+ self.batch_size = 1
+
+ def __iter__(self):
+ for i in range(2):
+ yield torch.ones([1, 10], dtype=torch.long)
+
+
+class TestAutoRound(unittest.TestCase):
+ @classmethod
+ def setUpClass(self):
+ self.model_name = "/tf_dataset/auto_round/models/facebook/opt-125m"
+ self.save_dir = ".saved/"
+ self.model = AutoModelForCausalLM.from_pretrained(self.model_name, torch_dtype="auto", trust_remote_code=True)
+ self.tokenizer = AutoTokenizer.from_pretrained(self.model_name, trust_remote_code=True)
+ self.llm_dataloader = LLMDataLoader()
+
+ @classmethod
+ def tearDownClass(self):
+ shutil.rmtree("./saved", ignore_errors=True)
+ shutil.rmtree("runs", ignore_errors=True)
+
+ @require_gptqmodel
+ def test_mixed_gptqmodel(self):
+ layer_config = {
+ "k_proj": {"bits": 8}, # part name
+ "lm_head": {"bits": 4}, # set lm_head quant
+ "fc1": {"bits": 16},
+ "model.decoder.layers.0.self_attn.v_proj": {"bits": 16},
+ "model.decoder.layers.0.self_attn.q_proj": {"bits": 8}, # full name
+ }
+ autoround = AutoRound(
+ model=self.model_name,
+ scheme="W4A16",
+ iters=2,
+ seqlen=2,
+ layer_config=layer_config,
+ dataset=self.llm_dataloader,
+ )
+ quantized_model_path = self.save_dir
+ autoround.quantize_and_save(output_dir=quantized_model_path, format="auto_gptq")
+ # test original GPTQModel inference
+ from gptqmodel import GPTQModel
+
+ model = GPTQModel.load(quantized_model_path)
+ assert model.model.model.decoder.layers[0].self_attn.k_proj.bits == 8
+ assert model.model.model.decoder.layers[0].self_attn.q_proj.bits == 8
+ assert model.model.model.decoder.layers[1].self_attn.v_proj.bits == 4
+ result = model.generate("Uncovering deep insights begins with")[0] # tokens
+ assert "!!!" not in model.tokenizer.decode(result) # string output
+ shutil.rmtree(quantized_model_path, ignore_errors=True)
+
+ def test_mixed_gptqmodel_convert_to_ar(self):
+ layer_config = {
+ "k_proj": {"bits": 8}, # part name
+ "lm_head": {"bits": 4}, # set lm_head quant
+ "fc1": {"bits": 16},
+ "model.decoder.layers.0.self_attn.v_proj": {"bits": 16},
+ "model.decoder.layers.0.self_attn.q_proj": {"bits": 8}, # full name
+ }
+ autoround = AutoRound(
+ model=self.model_name,
+ scheme="W4A16",
+ iters=2,
+ seqlen=2,
+ layer_config=layer_config,
+ dataset=self.llm_dataloader,
+ )
+ quantized_model_path = self.save_dir
+ autoround.quantize_and_save(output_dir=quantized_model_path, format="auto_gptq")
+ quantization_config = AutoRoundConfig()
+ model = AutoModelForCausalLM.from_pretrained(
+ quantized_model_path, device_map="cpu", quantization_config=quantization_config
+ )
+ tokenizer = AutoTokenizer.from_pretrained(quantized_model_path)
+ text = "There is a girl who likes adventure,"
+ inputs = tokenizer(text, return_tensors="pt").to(model.device)
+ res = tokenizer.decode(model.generate(**inputs, max_new_tokens=5)[0])
+ print(res)
+ shutil.rmtree(quantized_model_path, ignore_errors=True)
+
+ def test_mixed_autoround_format(self):
+ layer_config = {
+ "k_proj": {"bits": 8},
+ "q_proj": {"bits": 3},
+ "lm_head": {"bits": 16},
+ "fc1": {"bits": 16},
+ }
+ autoround = AutoRound(
+ model=self.model_name,
+ scheme="W4A16",
+ iters=2,
+ seqlen=2,
+ dataset=self.llm_dataloader,
+ layer_config=layer_config,
+ )
+ quantized_model_path = "./saved"
+ compressed_model = autoround.quantize_and_save(output_dir=quantized_model_path, format="auto_round")
+ model = AutoModelForCausalLM.from_pretrained(quantized_model_path, device_map="cpu")
+ assert model.model.decoder.layers[0].self_attn.k_proj.bits == 8
+ assert model.model.decoder.layers[0].self_attn.q_proj.bits == 3
+ tokenizer = AutoTokenizer.from_pretrained(quantized_model_path)
+ text = "There is a girl who likes adventure,"
+ inputs = tokenizer(text, return_tensors="pt").to(model.device)
+ print(tokenizer.decode(model.generate(**inputs, max_new_tokens=50)[0]))
+ shutil.rmtree(quantized_model_path, ignore_errors=True)
+
+ def test_fallback_regex_for_awq_format(self):
+ layer_config = {
+ "lm_head": {"bits": 16},
+ "fc1": {"bits": 16},
+ }
+ autoround = AutoRound(
+ model=self.model_name,
+ scheme="W4A16",
+ iters=2,
+ seqlen=2,
+ dataset=self.llm_dataloader,
+ layer_config=layer_config,
+ )
+ quantized_model_path = "./saved"
+ autoround.quantize_and_save(output_dir=quantized_model_path, format="auto_awq")
+ quantization_config = AutoRoundConfig()
+ model = AutoModelForCausalLM.from_pretrained(
+ quantized_model_path, device_map="cpu", quantization_config=quantization_config
+ )
+ tokenizer = AutoTokenizer.from_pretrained(quantized_model_path)
+ text = "There is a girl who likes adventure,"
+ inputs = tokenizer(text, return_tensors="pt").to(model.device)
+ print(tokenizer.decode(model.generate(**inputs, max_new_tokens=50)[0]))
+ shutil.rmtree(quantized_model_path, ignore_errors=True)
+
+ def test_mixed_ar_format_part_name_hf_loading(self):
+ layer_config = {
+ "k_proj": {"bits": 8}, # part name
+ "lm_head": {"bits": 16}, # full name
+ ".*fc1.*": {"bits": 16}, # standard regex
+ }
+ autoround = AutoRound(
+ model=self.model_name,
+ scheme="W4A16",
+ iters=2,
+ seqlen=2,
+ dataset=self.llm_dataloader,
+ layer_config=layer_config,
+ )
+ quantized_model_path = "./saved"
+ autoround.quantize()
+ autoround.save_quantized(output_dir=quantized_model_path, format="auto_round")
+ # remove old extra_config(which contains full name layer configs), only test regex config loading
+ new_extra_config = {
+ ".*fc1.*": { # standard regex
+ "act_bits": 16,
+ "act_data_type": "float",
+ "act_dynamic": True,
+ "act_group_size": 128,
+ "act_sym": True,
+ "bits": 16,
+ "data_type": "int",
+ "group_size": 128,
+ "sym": True,
+ },
+ "k_proj": { # part name
+ "act_bits": 16,
+ "act_data_type": "float",
+ "act_dynamic": True,
+ "act_group_size": 128,
+ "act_sym": True,
+ "bits": 8,
+ "data_type": "int",
+ "group_size": 128,
+ "sym": True,
+ },
+ }
+ config_file = Path(quantized_model_path) / "config.json"
+ with open(config_file, "r", encoding="utf-8") as f:
+ config = json.load(f)
+ quant_config = config.get("quantization_config", {})
+ old_extra_config = quant_config.get("extra_config", {})
+ quant_config["extra_config"] = new_extra_config
+ config["quantization_config"] = quant_config
+ with open(config_file, "w", encoding="utf-8") as f:
+ json.dump(config, f, indent=2, ensure_ascii=False)
+
+ model = AutoModelForCausalLM.from_pretrained(quantized_model_path, device_map="cpu")
+ assert model.model.decoder.layers[0].self_attn.k_proj.bits == 8
+ assert model.model.decoder.layers[0].self_attn.q_proj.bits == 4
+ tokenizer = AutoTokenizer.from_pretrained(quantized_model_path)
+ text = "There is a girl who likes adventure,"
+ inputs = tokenizer(text, return_tensors="pt").to(model.device)
+ print(tokenizer.decode(model.generate(**inputs, max_new_tokens=50)[0]))
+ shutil.rmtree(quantized_model_path, ignore_errors=True)
+
+ def test_mixed_MXFP_autoround_format_loading(self):
+ layer_config = {
+ "k_proj": {"bits": 8, "act_bits": 8},
+ "lm_head": {"bits": 16, "act_bits": 16},
+ "fc1": {"bits": 8, "act_bits": 8},
+ }
+ autoround = AutoRound(
+ self.model_name,
+ scheme="MXFP4",
+ iters=2,
+ seqlen=2,
+ dataset=self.llm_dataloader,
+ layer_config=layer_config,
+ )
+ quantized_model_path = self.save_dir
+ autoround.quantize_and_save(output_dir=quantized_model_path, inplace=False, format="auto_round")
+ model = AutoModelForCausalLM.from_pretrained(
+ quantized_model_path,
+ torch_dtype="auto",
+ device_map="cpu",
+ )
+ tokenizer = AutoTokenizer.from_pretrained(quantized_model_path)
+ from auto_round.eval.evaluation import simple_evaluate_user_model
+
+ result = simple_evaluate_user_model(model, tokenizer, batch_size=16, tasks="lambada_openai", limit=10)
+ print(result["results"]["lambada_openai"]["acc,none"])
+ self.assertGreater(result["results"]["lambada_openai"]["acc,none"], 0.14)
+ shutil.rmtree(quantized_model_path, ignore_errors=True)
+
+
+if __name__ == "__main__":
+ unittest.main()
diff --git a/test/test_cuda/test_mix_bits.py b/test/test_cuda/test_mix_bits.py
new file mode 100644
index 000000000..4f7d39d8c
--- /dev/null
+++ b/test/test_cuda/test_mix_bits.py
@@ -0,0 +1,329 @@
+import json
+import os
+import shutil
+import sys
+import unittest
+
+from parameterized import parameterized
+
+sys.path.insert(0, "../..")
+from pathlib import Path
+
+import torch
+from transformers import AutoModelForCausalLM, AutoRoundConfig, AutoTokenizer
+
+from auto_round import AutoRound
+from auto_round.testing_utils import (
+ require_awq,
+ require_gptqmodel,
+ require_package_version_ut,
+)
+
+
+class LLMDataLoader:
+ def __init__(self):
+ self.batch_size = 1
+
+ def __iter__(self):
+ for i in range(2):
+ yield torch.ones([1, 10], dtype=torch.long)
+
+
+class TestAutoRound(unittest.TestCase):
+ @classmethod
+ def setUpClass(self):
+ self.model_name = "/models/opt-125m"
+ self.save_dir = "./saved"
+ self.model = AutoModelForCausalLM.from_pretrained(self.model_name, torch_dtype="auto", trust_remote_code=True)
+ self.tokenizer = AutoTokenizer.from_pretrained(self.model_name, trust_remote_code=True)
+ self.llm_dataloader = LLMDataLoader()
+
+ @classmethod
+ def tearDownClass(self):
+ shutil.rmtree("./saved", ignore_errors=True)
+ shutil.rmtree("runs", ignore_errors=True)
+
+ @require_gptqmodel
+ def test_mixed_gptqmodel(self):
+ scheme = "W4A16"
+ layer_config = {
+ "k_proj": {"bits": 8}, # part name
+ "lm_head": {"bits": 4}, # set lm_head quant
+ "fc1": {"bits": 16},
+ "model.decoder.layers.0.self_attn.v_proj": {"bits": 16},
+ "model.decoder.layers.0.self_attn.q_proj": {"bits": 8}, # full name
+ }
+ autoround = AutoRound(
+ model=self.model_name,
+ scheme=scheme,
+ iters=2,
+ seqlen=2,
+ layer_config=layer_config,
+ dataset=self.llm_dataloader,
+ )
+ quantized_model_path = self.save_dir
+ autoround.quantize_and_save(output_dir=quantized_model_path, format="auto_gptq")
+ from gptqmodel import GPTQModel
+
+ model = GPTQModel.load(quantized_model_path)
+ assert model.model.model.decoder.layers[0].self_attn.k_proj.bits == 8
+ assert model.model.model.decoder.layers[0].self_attn.q_proj.bits == 8
+ assert model.model.model.decoder.layers[1].self_attn.v_proj.bits == 4
+ res = model.tokenizer.decode(model.generate("Uncovering deep insights begins with")[0])
+ print(res)
+ shutil.rmtree(quantized_model_path, ignore_errors=True)
+
+ def test_mixed_gptqmodel_convert_to_ar(self):
+ layer_config = {
+ "k_proj": {"bits": 8}, # part name
+ "lm_head": {"bits": 4}, # set lm_head quant
+ "model.decoder.layers.0.self_attn.v_proj": {"bits": 16},
+ "model.decoder.layers.0.self_attn.q_proj": {"bits": 8}, # full name
+ }
+ autoround = AutoRound(
+ model=self.model_name,
+ scheme="W4A16",
+ iters=2,
+ seqlen=2,
+ layer_config=layer_config,
+ dataset=self.llm_dataloader,
+ )
+ quantized_model_path = self.save_dir
+ autoround.quantize_and_save(output_dir=quantized_model_path, format="auto_gptq")
+ quantization_config = AutoRoundConfig()
+ model = AutoModelForCausalLM.from_pretrained(
+ quantized_model_path, device_map="auto", quantization_config=quantization_config
+ )
+ tokenizer = AutoTokenizer.from_pretrained(quantized_model_path)
+ text = "There is a girl who likes adventure,"
+ inputs = tokenizer(text, return_tensors="pt").to(model.device)
+ res = tokenizer.decode(model.generate(**inputs, max_new_tokens=5)[0])
+ print(res)
+ shutil.rmtree(quantized_model_path, ignore_errors=True)
+
+ def test_mixed_autoround_format(self):
+ layer_config = {
+ "k_proj": {"bits": 8},
+ "q_proj": {"bits": 3},
+ "lm_head": {"bits": 16},
+ "fc1": {"bits": 16},
+ }
+ autoround = AutoRound(
+ model=self.model_name,
+ scheme="W4A16",
+ iters=2,
+ seqlen=2,
+ dataset=self.llm_dataloader,
+ layer_config=layer_config,
+ )
+ quantized_model_path = "self.save_dir"
+ autoround.quantize_and_save(output_dir=quantized_model_path, format="auto_round")
+ model = AutoModelForCausalLM.from_pretrained(quantized_model_path, device_map="auto")
+ assert model.model.decoder.layers[0].self_attn.k_proj.bits == 8
+ assert model.model.decoder.layers[0].self_attn.q_proj.bits == 3
+ tokenizer = AutoTokenizer.from_pretrained(quantized_model_path)
+ text = "There is a girl who likes adventure,"
+ inputs = tokenizer(text, return_tensors="pt").to(model.device)
+ print(tokenizer.decode(model.generate(**inputs, max_new_tokens=50)[0]))
+ shutil.rmtree(quantized_model_path, ignore_errors=True)
+
+ @require_awq
+ @require_package_version_ut("transformers", "<4.57.0")
+ def test_fallback_regex_for_awq_format(self):
+ model_name = "facebook/opt-125m"
+ layer_config = {
+ "lm_head": {"bits": 16},
+ "fc1": {"bits": 16},
+ }
+ autoround = AutoRound(
+ model=model_name,
+ scheme="W4A16",
+ iters=2,
+ seqlen=2,
+ dataset=self.llm_dataloader,
+ layer_config=layer_config,
+ )
+ quantized_model_path = "self.save_dir"
+ autoround.quantize_and_save(output_dir=quantized_model_path, format="auto_awq")
+ quantization_config = AutoRoundConfig()
+ model = AutoModelForCausalLM.from_pretrained(
+ quantized_model_path, device_map="auto", quantization_config=quantization_config
+ )
+ tokenizer = AutoTokenizer.from_pretrained(quantized_model_path)
+ text = "There is a girl who likes adventure,"
+ inputs = tokenizer(text, return_tensors="pt").to(model.device)
+ print(tokenizer.decode(model.generate(**inputs, max_new_tokens=50)[0]))
+ shutil.rmtree(quantized_model_path, ignore_errors=True)
+
+ def test_mixed_ar_format_part_name_hf_loading(self):
+ layer_config = {
+ "k_proj": {"bits": 8}, # part name
+ "lm_head": {"bits": 16}, # full name
+ ".*fc1.*": {"bits": 16}, # standard regex
+ }
+ autoround = AutoRound(
+ model=self.model_name,
+ scheme="W4A16",
+ iters=2,
+ seqlen=2,
+ dataset=self.llm_dataloader,
+ layer_config=layer_config,
+ )
+ quantized_model_path = "self.save_dir"
+ autoround.quantize()
+ autoround.save_quantized(output_dir=quantized_model_path, format="auto_round")
+ # remove old extra_config(which contains full name layer configs), only test regex config loading
+ new_extra_config = {
+ ".*fc1.*": { # standard regex
+ "act_bits": 16,
+ "act_data_type": "float",
+ "act_dynamic": True,
+ "act_group_size": 128,
+ "act_sym": True,
+ "bits": 16,
+ "data_type": "int",
+ "group_size": 128,
+ "sym": True,
+ },
+ "k_proj": { # part name
+ "act_bits": 16,
+ "act_data_type": "float",
+ "act_dynamic": True,
+ "act_group_size": 128,
+ "act_sym": True,
+ "bits": 8,
+ "data_type": "int",
+ "group_size": 128,
+ "sym": True,
+ },
+ }
+ config_file = Path(quantized_model_path) / "config.json"
+ with open(config_file, "r", encoding="utf-8") as f:
+ config = json.load(f)
+ quant_config = config.get("quantization_config", {})
+ old_extra_config = quant_config.get("extra_config", {})
+ quant_config["extra_config"] = new_extra_config
+ config["quantization_config"] = quant_config
+ with open(config_file, "w", encoding="utf-8") as f:
+ json.dump(config, f, indent=2, ensure_ascii=False)
+
+ model = AutoModelForCausalLM.from_pretrained(quantized_model_path, device_map="auto")
+ assert model.model.decoder.layers[0].self_attn.k_proj.bits == 8
+ assert model.model.decoder.layers[0].self_attn.q_proj.bits == 4
+ tokenizer = AutoTokenizer.from_pretrained(quantized_model_path)
+ text = "There is a girl who likes adventure,"
+ inputs = tokenizer(text, return_tensors="pt").to(model.device)
+ print(tokenizer.decode(model.generate(**inputs, max_new_tokens=50)[0]))
+ shutil.rmtree(quantized_model_path, ignore_errors=True)
+
+ def test_mixed_MXFP_autoround_format_loading(self):
+ layer_config = {
+ "k_proj": {"bits": 8, "act_bits": 8},
+ "lm_head": {"bits": 16, "act_bits": 16},
+ "fc1": {"bits": 8, "act_bits": 8},
+ }
+ autoround = AutoRound(
+ self.model_name,
+ scheme="MXFP4",
+ iters=2,
+ seqlen=2,
+ dataset=self.llm_dataloader,
+ layer_config=layer_config,
+ )
+ quantized_model_path = self.save_dir
+ autoround.quantize_and_save(output_dir=quantized_model_path, inplace=False, format="auto_round")
+ model = AutoModelForCausalLM.from_pretrained(
+ quantized_model_path,
+ torch_dtype="auto",
+ device_map="auto",
+ )
+ tokenizer = AutoTokenizer.from_pretrained(quantized_model_path)
+ from auto_round.eval.evaluation import simple_evaluate_user_model
+
+ result = simple_evaluate_user_model(model, tokenizer, batch_size=16, tasks="lambada_openai")
+ print(result["results"]["lambada_openai"]["acc,none"])
+ self.assertGreater(result["results"]["lambada_openai"]["acc,none"], 0.32)
+ shutil.rmtree(quantized_model_path, ignore_errors=True)
+
+ def test_mixed_autoround_format_vllm(self):
+ layer_config = {
+ "self_attn": {"bits": 8},
+ "lm_head": {"bits": 16},
+ }
+ autoround = AutoRound(
+ self.model,
+ self.tokenizer,
+ scheme="W4A16",
+ iters=2,
+ seqlen=2,
+ dataset=self.llm_dataloader,
+ layer_config=layer_config,
+ )
+ autoround.quantize()
+ quantized_model_path = self.save_dir
+ autoround.save_quantized(output_dir=quantized_model_path, inplace=False, format="auto_round")
+
+ from vllm import LLM, SamplingParams
+
+ # Sample prompts.
+ prompts = [
+ "The capital of France is",
+ "The future of AI is",
+ ]
+ # Create a sampling params object.
+ sampling_params = SamplingParams(temperature=0.8, top_p=0.95)
+ # Create an LLM.
+ QUANTIZATION = "auto-round" # quantized_model_path
+ llm = LLM(model=quantized_model_path, quantization=QUANTIZATION, trust_remote_code=True, tensor_parallel_size=1)
+ outputs = llm.generate(prompts, sampling_params)
+ # Print the outputs.
+ for output in outputs:
+ prompt = output.prompt
+ generated_text = output.outputs[0].text
+ # if "France" in prompt:
+ assert "!!!" not in generated_text
+ print(f"{prompt}: {generated_text}")
+ shutil.rmtree(quantized_model_path, ignore_errors=True)
+
+ def test_mixed_llmcompressor_format_vllm(self):
+ layer_config = {
+ "self_attn": {"bits": 16, "act_bits": 16},
+ "lm_head": {"bits": 16, "act_bits": 16},
+ "fc1": {"bits": 16, "act_bits": 16},
+ }
+ autoround = AutoRound(
+ self.model_name,
+ scheme="NVFP4",
+ iters=2,
+ seqlen=2,
+ dataset=self.llm_dataloader,
+ layer_config=layer_config,
+ )
+ quantized_model_path = self.save_dir
+ compressed, _ = autoround.quantize_and_save(
+ output_dir=quantized_model_path, inplace=False, format="llm_compressor"
+ )
+ from vllm import LLM, SamplingParams
+
+ # Sample prompts.
+ prompts = [
+ "The capital of France is",
+ "The future of AI is",
+ ]
+ # Create a sampling params object.
+ sampling_params = SamplingParams(temperature=0.8, top_p=0.95)
+ # Create an LLM.
+ QUANTIZATION = "auto-round" # quantized_model_path
+ llm = LLM(model=quantized_model_path, trust_remote_code=True, tensor_parallel_size=1)
+ outputs = llm.generate(prompts, sampling_params)
+ # Print the outputs.
+ for output in outputs:
+ prompt = output.prompt
+ generated_text = output.outputs[0].text
+ print(f"{prompt}: {generated_text}")
+ assert "!!!" not in generated_text
+ shutil.rmtree(quantized_model_path, ignore_errors=True)
+
+
+if __name__ == "__main__":
+ unittest.main()