Skip to content

PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度

rename llmcompressor to llm_compressor for align with other formats (#780)

Weiweie66652a4d624
diff --git a/auto_round/autoround.py b/auto_round/autoround.py
index 7a087558d..c22479ab7 100644
--- a/auto_round/autoround.py
+++ b/auto_round/autoround.py
@@ -691,7 +691,7 @@ class AutoRound(object):
if self.gradient_accumulate_steps <= 0:
raise ValueError("`gradient_accumulate_steps` must be positive")
- if self.act_bits <= 8:
+ if self.act_bits <= 8 and (not is_nv_fp(self.act_data_type) or "static_gs" not in self.act_data_type):
logger.warning(
"activation quantization is an experimental feature with limited support and a complex API. "
"And please save the quantized model to fake format as real deployment is not supported currently"
@@ -843,19 +843,21 @@ class AutoRound(object):
"for the current quantization configuration, "
"please change to `fake` format for research purpose"
)
-
formats[index] = format
- elif format == "llmcompressor":
+ elif format == "llm_compressor":
from auto_round.export.export_to_llmcompressor import check_compressed_tensors_supported
if check_compressed_tensors_supported() and (is_nv_fp(self.data_type) or is_mx_fp(self.data_type)):
- format = format.replace("llmcompressor", f"llmcompressor:{self.data_type}")
+ format = format.replace("llm_compressor", f"llm_compressor:{self.data_type}")
formats[index] = format
elif not is_wfp8afp8(self):
logger.error(
- "Currently, the llmcompressor format only supports MXFP/NVFP/FP8. "
+ "Currently, the llm_compressor format only supports MXFP/NVFP/FP8. "
"Please change format to fake or auto_round etc."
)
+ else:
+ if (is_nv_fp(self.data_type) or is_mx_fp(self.data_type)) and format != "fake":
+ logger.warning(f"nv_fp and mx_fp dtypes are not supported for export format: {format}")
# Remove duplicates from formats list
def remove_duplicates(lst):
@@ -887,8 +889,13 @@ class AutoRound(object):
# Only support to export afp8/nv_fp
if self.act_bits <= 8:
if not is_standard_fp(self.act_data_type) or self.act_dynamic:
- if format == "llmcompressor":
- if is_nv_fp(self.act_data_type):
+ if "llm_compressor" in format:
+ if is_nv_fp(self.act_data_type) and "static_gs" in self.act_data_type:
+ logger.warning(
+ f"AutoRound supports exporting to format '{format}', "
+ "but loading quantized models in this format is not yet supported. "
+ "It is currently recommended to export to the 'llm_compressor' format."
+ )
return format
bits, group_size, sym, act_bits = 8, -1, True, 8
assert (
@@ -899,10 +906,11 @@ class AutoRound(object):
and self.act_dynamic
), (
f"Currently only support to export llmcompressor format for dynamic quantized"
- f" W{self.bits}A{self.act_bits} model, but got bits={self.bits},"
- f" group_size={self.group_size}, sym={self.sym}, act_bits={self.act_bits}"
+ f" W{bits}Afp{act_bits} model, but got bits={self.bits}, data_type={self.data_type}"
+ f" group_size={self.group_size}, sym={self.sym}"
+ f", act_bits={self.act_bits}, act_data_type={self.act_data_type}"
)
- elif format != "fake" and not is_nv_fp(format):
+ elif format != "fake" and (not is_nv_fp(format) or "static_gs" not in self.act_data_type):
logger.warning(
"Currently only support to export auto_round format quantized model"
" with fp8 or nv_fp4 dtype activation for activation quantization."
@@ -1652,7 +1660,7 @@ class AutoRound(object):
or "gptq" in formats[0]
or "auto_round" in formats[0]
or "gguf" in formats[0]
- or "llmcompressor" in formats[0]
+ or "llm_compressor" in formats[0]
)
and self.inplace
):
@@ -3017,8 +3025,8 @@ class AutoRound(object):
"Support for exporting activation quantization is limited. "
"Please ensure that your configuration is supported."
)
- if format == "llmcompressor" and (is_nv_fp(self.data_type) or is_mx_fp(self.data_type)):
- format = format.replace("llmcompressor", f"llmcompressor:{self.data_type}")
+ if format == "llm_compressor" and (is_nv_fp(self.data_type) or is_mx_fp(self.data_type)):
+ format = format.replace("llm_compressor", f"llm_compressor:{self.data_type}")
from auto_round.export import EXPORT_FORMAT
diff --git a/auto_round/export/__init__.py b/auto_round/export/__init__.py
index 5956238d6..db20c94d8 100644
--- a/auto_round/export/__init__.py
+++ b/auto_round/export/__init__.py
@@ -78,14 +78,14 @@ def _packing_layer_with_autoawq(*args, **kwargs):
return pack_layer(*args, **kwargs)
-@register_format("llmcompressor")
+@register_format("llm_compressor")
def _save_quantized_as_llmcompressor(*args, **kwargs):
from auto_round.export.export_to_llmcompressor.export import save_quantized_as_llmcompressor
return save_quantized_as_llmcompressor(*args, **kwargs)
-@register_layer_packing("llmcompressor")
+@register_layer_packing("llm_compressor")
def _packing_layer_with_llmcompressor(*args, **kwargs):
from auto_round.export.export_to_llmcompressor.export import pack_layer
diff --git a/auto_round/export/export_to_autoround/export.py b/auto_round/export/export_to_autoround/export.py
index e5f195609..6a6c34c88 100644
--- a/auto_round/export/export_to_autoround/export.py
+++ b/auto_round/export/export_to_autoround/export.py
@@ -25,6 +25,7 @@ import torch.nn as nn
import transformers
from tqdm import tqdm
+from auto_round.export.export_to_autoround.utils import REQUIRED_CONFIG_KEYS, check_neq_config
from auto_round.utils import (
SUPPORTED_FORMATS,
SUPPORTED_LAYER_TYPES,
@@ -40,8 +41,6 @@ from auto_round.utils import (
set_module,
)
-from .utils import check_neq_config
-
def dynamic_import_quant_linear_for_packing(backend, bits, group_size, sym, act_bits=16):
"""
@@ -313,12 +312,7 @@ def save_quantized_as_autoround(output_dir, inplace=True, backend="auto_round:ex
block_name_to_quantize is not None and check_start_with_block_name(layer_name, block_name_to_quantize)
):
neq_keys = check_neq_config(
- layer_config[layer_name],
- data_type=quantization_config["data_type"],
- bits=quantization_config["bits"],
- act_bits=quantization_config["act_bits"],
- group_size=quantization_config["group_size"],
- sym=quantization_config["sym"],
+ layer_config[layer_name], **{k: quantization_config[k] for k in REQUIRED_CONFIG_KEYS}
)
if len(neq_keys) > 0:
extra_config[layer_name] = {}
diff --git a/auto_round/export/export_to_autoround/export_to_fp8.py b/auto_round/export/export_to_autoround/export_to_fp8.py
index b8a32896f..af196d7b6 100644
--- a/auto_round/export/export_to_autoround/export_to_fp8.py
+++ b/auto_round/export/export_to_autoround/export_to_fp8.py
@@ -23,8 +23,10 @@ import transformers
from tqdm import tqdm
from auto_round.data_type.utils import reshape_pad_tensor_by_group_size, revert_tensor_by_pad
+from auto_round.export.export_to_autoround.utils import REQUIRED_CONFIG_KEYS, check_neq_config
from auto_round.utils import (
SUPPORTED_LAYER_TYPES,
+ _get_device,
check_start_with_block_name,
check_to_quantized,
filter_quantization_config,
@@ -33,8 +35,6 @@ from auto_round.utils import (
set_module,
)
-from .utils import check_neq_config
-
class FP8WOQLinear(torch.nn.Module):
@@ -86,11 +86,7 @@ def pack_layer(layer_name, model, data_type, packing_device=None):
None: The function modifies the model in place.
"""
if packing_device is None:
- packing_device = "cpu"
- if torch.cuda.is_available():
- packing_device = "cuda"
- elif torch.xpu.is_available():
- packing_device = "xpu"
+ packing_device = _get_device()
layer = get_module(model, layer_name)
if hasattr(layer, "orig_layer"):
layer = layer.orig_layer
@@ -187,12 +183,7 @@ def save_quantized_as_autoround(output_dir, inplace=True, backend="auto_round",
block_name_to_quantize is not None and check_start_with_block_name(layer_name, block_name_to_quantize)
):
neq_keys = check_neq_config(
- layer_config[layer_name],
- data_type=quantization_config["data_type"],
- bits=quantization_config["bits"],
- act_bits=quantization_config["act_bits"],
- group_size=quantization_config["group_size"],
- sym=quantization_config["sym"],
+ layer_config[layer_name], **{k: quantization_config[k] for k in REQUIRED_CONFIG_KEYS}
)
if len(neq_keys) > 0:
extra_config[layer_name] = {}
@@ -205,11 +196,7 @@ def save_quantized_as_autoround(output_dir, inplace=True, backend="auto_round",
max_workers = 1
if not torch.cuda.is_available() and not torch.xpu.is_available():
max_workers = 2 ## 2 with cuda packing will cause hang occasionally
- packing_device = "cpu"
- if torch.cuda.is_available():
- packing_device = "cuda"
- elif torch.xpu.is_available():
- packing_device = "xpu"
+ packing_device = _get_device()
with ThreadPoolExecutor(max_workers=max_workers) as executor:
with tqdm(total=len(names), leave=True) as pbar:
diff --git a/auto_round/export/export_to_autoround/export_to_nvfp_mxfp.py b/auto_round/export/export_to_autoround/export_to_nvfp_mxfp.py
index 2c148e744..06ef32a02 100644
--- a/auto_round/export/export_to_autoround/export_to_nvfp_mxfp.py
+++ b/auto_round/export/export_to_autoround/export_to_nvfp_mxfp.py
@@ -24,6 +24,7 @@ import torch.nn as nn
import transformers
from tqdm import tqdm
+from auto_round.export.export_to_autoround.utils import REQUIRED_CONFIG_KEYS, check_neq_config
from auto_round.utils import (
SUPPORTED_LAYER_TYPES,
check_start_with_block_name,
@@ -39,7 +40,6 @@ from auto_round.utils import (
from auto_round.wrapper import WrapperWALayer
from .qlinear_fp import QuantLinear
-from .utils import check_neq_config
__all__ = [
"pack_layer",
@@ -203,12 +203,7 @@ def save_quantized_as_fp(output_dir, inplace=True, **kwargs):
block_name_to_quantize is not None and check_start_with_block_name(layer_name, block_name_to_quantize)
):
neq_keys = check_neq_config(
- layer_config[layer_name],
- data_type=quantization_config["data_type"],
- bits=quantization_config["bits"],
- act_bits=quantization_config["act_bits"],
- group_size=quantization_config["group_size"],
- sym=quantization_config["sym"],
+ layer_config[layer_name], **{k: quantization_config[k] for k in REQUIRED_CONFIG_KEYS}
)
if len(neq_keys) > 0:
extra_config[layer_name] = {}
diff --git a/auto_round/export/export_to_autoround/qlinear_fp.py b/auto_round/export/export_to_autoround/qlinear_fp.py
index f72fee72f..cff38a30a 100644
--- a/auto_round/export/export_to_autoround/qlinear_fp.py
+++ b/auto_round/export/export_to_autoround/qlinear_fp.py
@@ -38,7 +38,7 @@ import transformers
from auto_round.data_type.mxfp import FP32_EXPONENT_BIAS, FP32_MIN_NORMAL
from auto_round.data_type.nvfp import cast_to_fp4, get_reciprocal
from auto_round.data_type.utils import reshape_pad_tensor_by_group_size, revert_tensor_by_pad
-from auto_round.utils import is_mx_fp, is_nv_fp
+from auto_round.utils import _get_device, is_mx_fp, is_nv_fp
# from auto_round.utils import get_weight_compress_dtype
logger = getLogger(__name__)
@@ -141,15 +141,11 @@ class QuantLinear(nn.Module):
pass
def pack(self, linear, scales, zeros=None, g_idx=None, global_scale=None, input_global_scale=None):
- if linear.bias is not None:
- self.bias = linear.bias.clone().half()
- device = "cpu"
- if torch.cuda.is_available():
- device = "cuda:0"
- elif torch.xpu.is_available():
- device = "xpu:0"
-
- W = linear.weight.data.to(device).clone()
+ device = _get_device()
+ if getattr(linear, "bias", None) is not None:
+ self.bias = linear.bias.detach().to(torch.float16)
+
+ W = linear.weight.data.detach().to(device)
if isinstance(linear, nn.Conv2d):
W = W.flatten(1)
if isinstance(linear, transformers.pytorch_utils.Conv1D):
@@ -163,7 +159,8 @@ class QuantLinear(nn.Module):
scaled_tensor = tensor.to(global_scale.dtype) * get_reciprocal(
scales.reshape(tensor.shape[0], -1) * get_reciprocal(global_scale)
)
- scaled_tensor = cast_to_fp4(torch.clamp(scaled_tensor, -6.0, 6.0))
+ scaled_tensor.clamp_(-6.0, 6.0)
+ scaled_tensor = cast_to_fp4(scaled_tensor)
else:
scaled_tensor = tensor / (2 ** scales.reshape(tensor.shape[0], -1))
scaled_tensor = revert_tensor_by_pad(scaled_tensor, orig_shape=orig_shape, pad_len=pad_len)
diff --git a/auto_round/export/export_to_autoround/qlinear_triton_act.py b/auto_round/export/export_to_autoround/qlinear_triton_act.py
index 21fc99642..ed06024ec 100644
--- a/auto_round/export/export_to_autoround/qlinear_triton_act.py
+++ b/auto_round/export/export_to_autoround/qlinear_triton_act.py
@@ -41,6 +41,8 @@ import torch
import torch.nn as nn
import transformers
+from auto_round.utils import _get_device
+
logger = getLogger(__name__)
@@ -117,6 +119,7 @@ class QuantLinear(nn.Module):
pass
def pack(self, linear, scales, zeros, act_scales, w_bf16_to_fp8_scale, g_idx=None):
+ device = _get_device()
scales_t = scales.t().contiguous()
self.act_scales.data.copy_(act_scales.squeeze().clone())
@@ -124,9 +127,6 @@ class QuantLinear(nn.Module):
if linear.bias is not None:
self.bias = linear.bias.clone().half()
self.scales = scales_t.clone().half()
- device = "cpu"
- if torch.cuda.is_available():
- device = "cuda:0"
W = linear.weight.data.to(device).clone()
if isinstance(linear, nn.Conv2d):
diff --git a/auto_round/export/export_to_autoround/utils.py b/auto_round/export/export_to_autoround/utils.py
index 11b217424..19bd92f43 100644
--- a/auto_round/export/export_to_autoround/utils.py
+++ b/auto_round/export/export_to_autoround/utils.py
@@ -12,26 +12,36 @@
# See the License for the specific language governing permissions and
# limitations under the License.
+REQUIRED_CONFIG_KEYS = (
+ "data_type",
+ "bits",
+ "group_size",
+ "sym",
+ "act_bits",
+ "act_data_type",
+ "act_group_size",
+ "act_sym",
+ "act_dynamic",
+)
-def check_neq_config(config, data_type, bits, act_bits, group_size, sym):
- """
- Checks if the provided configuration parameters are not equal to the values in the config dictionary.
- Args:
- config (dict): A dictionary containing the configuration parameters.
- data_type (str): The expected data type.
- bits (int): The expected number of bits.
- group_size (int): The expected group size.
- sym (bool): The expected symmetry flag.
+def check_neq_config(config: dict, **expected) -> dict[str, tuple]:
+ """
+ Compare a config dict against expected values.
+ Ensures all required keys are present in both config and expected.
Returns:
- list: A list of strings indicating which configuration parameters do not match.
+ dict[str, tuple]: {key: (actual, expected)} for mismatched values.
"""
- expected_config = {
- "data_type": data_type,
- "bits": bits,
- "group_size": group_size,
- "sym": sym,
- "act_bits": act_bits,
- }
- return [key for key, expected_value in expected_config.items() if config.get(key) != expected_value]
+ # 1. Check missing from expected
+ missing_expected = [k for k in REQUIRED_CONFIG_KEYS if k not in expected]
+ if missing_expected:
+ raise ValueError(f"Missing expected values for keys: {missing_expected}")
+
+ # 2. Check missing from layer config
+ missing_config = [k for k in REQUIRED_CONFIG_KEYS if k not in config]
+ if missing_config:
+ raise ValueError(f"Missing config values for keys: {missing_config}")
+
+ # 3. Collect mismatches
+ return {key: (config[key], expected[key]) for key in REQUIRED_CONFIG_KEYS if config[key] != expected[key]}
diff --git a/auto_round/export/export_to_llmcompressor/export.py b/auto_round/export/export_to_llmcompressor/export.py
index 0f759384b..139d4423d 100644
--- a/auto_round/export/export_to_llmcompressor/export.py
+++ b/auto_round/export/export_to_llmcompressor/export.py
@@ -60,8 +60,8 @@ def pack_layer(layer_name, model, backend):
return pack_layer(layer_name, model, backend)
- ## passed as no other llmcompressor format is supported yet
- logger.warning("No other llmcompressor packing format(except NVFP&MXFP) is supported yet, skip packing")
+ ## passed as no other llm_compressor format is supported yet
+ logger.warning("No other llm_compressor packing format(except NVFP&MXFP) is supported yet, skip packing")
return
diff --git a/auto_round/script/llm.py b/auto_round/script/llm.py
index c1e55cf38..9f7319157 100644
--- a/auto_round/script/llm.py
+++ b/auto_round/script/llm.py
@@ -511,7 +511,7 @@ def tune(args):
"auto_round" not in format
and "fake" not in format
and "awq" not in format
- and "llmcompressor" not in format
+ and "llm_compressor" not in format
):
# TODO gptq could support some mixed precision config
logger.warning(f"mixed precision exporting does not support {format} currently")
diff --git a/auto_round/utils.py b/auto_round/utils.py
index eb67a35d5..355909979 100644
--- a/auto_round/utils.py
+++ b/auto_round/utils.py
@@ -54,7 +54,7 @@ class SupportedFormats:
"itrex",
"itrex_xpu",
"fake",
- "llmcompressor",
+ "llm_compressor",
)
self._gguf_format = tuple(sorted(GGUF_CONFIG.keys()))
self._support_list = self._support_format + self._gguf_format
@@ -2647,3 +2647,12 @@ def get_max_vram(ratio: float = 0.9) -> dict:
else:
raise RuntimeError("No CUDA or XPU devices found.")
return max_memory
+
+
+def _get_device() -> torch.device:
+ """Selects best available device (CUDA > XPU > CPU)."""
+ if torch.cuda.is_available():
+ return torch.device("cuda:0")
+ if hasattr(torch, "xpu") and torch.xpu.is_available():
+ return torch.device("xpu:0")
+ return torch.device("cpu")
diff --git a/docs/step_by_step.md b/docs/step_by_step.md
index bd440c657..3c1c69d1d 100644
--- a/docs/step_by_step.md
+++ b/docs/step_by_step.md
@@ -133,8 +133,7 @@ models. Besides, recently 3 bits may have some accuracy issues in Transformers.
**AutoAWQ Format**: This format is well-suited for asymmetric 4-bit quantization on CUDA devices and is widely
adopted within the community, **only 4-bits quantization is supported**.
-**llmcompressor Format**: This format is for reusing llmcompressor format, **only INT8 W8A8 dynamic quantization is
-supported**.
+**LlmCompressor Format**: **only INT8 W8A8 dynamic quantization is supported**.
### Hardware Compatibility
@@ -650,3 +649,4 @@ If not explicitly specify '--task', the default value will be used (typically co
* Random quantization results in tuning some models
* ChatGlm-V1 is not supported
+
diff --git a/test/test_cpu/test_export.py b/test/test_cpu/test_export.py
index 603f223c1..536cd338a 100644
--- a/test/test_cpu/test_export.py
+++ b/test/test_cpu/test_export.py
@@ -297,7 +297,7 @@ class TestAutoRound(unittest.TestCase):
quantized_model_path = self.save_dir
autoround.quantize()
compressed_model = autoround.save_quantized(
- output_dir=quantized_model_path, inplace=True, format="llmcompressor"
+ output_dir=quantized_model_path, inplace=True, format="llm_compressor"
)
tmp_layer = compressed_model.model.decoder.layers[3].self_attn.q_proj
skip_layer = compressed_model.model.decoder.layers[3].self_attn.k_proj
@@ -341,7 +341,7 @@ class TestAutoRound(unittest.TestCase):
dataset=self.llm_dataloader,
)
quantized_model_path = self.save_dir
- compressed_model, _ = autoround.quantize_and_save(output_dir=quantized_model_path, format="llmcompressor")
+ compressed_model, _ = autoround.quantize_and_save(output_dir=quantized_model_path, format="llm_compressor")
tmp_layer = compressed_model.model.decoder.layers[3].self_attn.q_proj
assert (
hasattr(tmp_layer, "weight_scale")
@@ -385,7 +385,7 @@ class TestAutoRound(unittest.TestCase):
dataset=self.llm_dataloader,
)
quantized_model_path = self.save_dir
- compressed_model, _ = autoround.quantize_and_save(output_dir=quantized_model_path, format="llmcompressor")
+ compressed_model, _ = autoround.quantize_and_save(output_dir=quantized_model_path, format="llm_compressor")
tmp_layer = compressed_model.model.decoder.layers[3].self_attn.q_proj
assert (
hasattr(tmp_layer, "weight_scale")
diff --git a/test/test_cpu/test_llmcompressor_w8a8.py b/test/test_cpu/test_llmcompressor_w8a8.py
index cbd3c2abb..b539ab5d0 100644
--- a/test/test_cpu/test_llmcompressor_w8a8.py
+++ b/test/test_cpu/test_llmcompressor_w8a8.py
@@ -38,7 +38,7 @@ class TestLLMC(unittest.TestCase):
iters=0,
)
autoround.quantize()
- autoround.save_quantized("./saved", format="llmcompressor", inplace=True)
+ autoround.save_quantized("./saved", format="llm_compressor", inplace=True)
if __name__ == "__main__":
diff --git a/test/test_cuda/test_export.py b/test/test_cuda/test_export.py
index 9e4711915..541b09a78 100644
--- a/test/test_cuda/test_export.py
+++ b/test/test_cuda/test_export.py
@@ -344,7 +344,7 @@ class TestAutoRound(unittest.TestCase):
dataset=self.llm_dataloader,
)
quantized_model_path = self.save_dir
- compressed_model, _ = autoround.quantize_and_save(output_dir=quantized_model_path, format="llmcompressor")
+ compressed_model, _ = autoround.quantize_and_save(output_dir=quantized_model_path, format="llm_compressor")
tmp_layer = compressed_model.model.decoder.layers[3].self_attn.q_proj
assert (
hasattr(tmp_layer, "weight_scale")