Skip to content

PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度

Fix GGUF-K RTN routing and fake eval dtype normalization (#1807)

Heng Guo351ba32ad8ff
diff --git a/auto_round/__main__.py b/auto_round/__main__.py
index a7e4a9569..42a230804 100644
--- a/auto_round/__main__.py
+++ b/auto_round/__main__.py
@@ -806,6 +806,7 @@ def tune(args):
autoround: BaseCompressor = AutoRound(
model=model_name,
platform=args.platform,
+ format=args.format,
scheme=scheme,
dataset=args.dataset,
iters=args.iters,
diff --git a/auto_round/auto_scheme/delta_loss.py b/auto_round/auto_scheme/delta_loss.py
index a7e2bdf90..792c71d10 100644
--- a/auto_round/auto_scheme/delta_loss.py
+++ b/auto_round/auto_scheme/delta_loss.py
@@ -66,6 +66,7 @@ __all__ = ["gen_layer_config"]
class AutoSchemeWrapperLinear(WrapperLinear):
+
def __init__(
self,
orig_layer,
@@ -167,6 +168,7 @@ class AutoSchemeWrapperLinear(WrapperLinear):
class AutoSchemeWrapperLinearForGGUFK(AutoSchemeWrapperLinear):
+
def __init__(
self,
orig_layer,
@@ -212,6 +214,7 @@ class AutoSchemeWrapperLinearForGGUFK(AutoSchemeWrapperLinear):
class AutoSchemeWrapperLinearForGGUFKImatrix(AutoSchemeWrapperLinear):
+
def __init__(
self,
orig_layer,
@@ -291,6 +294,7 @@ class AutoSchemeWrapperLinearForGGUFKImatrix(AutoSchemeWrapperLinear):
@torch.no_grad()
def cal_imatrix(model, dataloader):
+
def register_act_hook(model):
"""Registers hooks to accumulate activation squared norms into `imatrix`."""
@@ -923,6 +927,7 @@ def choose_bits_per_layer_with_path(layers: dict, P: int):
def move_module_to_tuning_device(module, major_device="cpu"):
+
def _normalize(dev):
return dev if isinstance(dev, torch.device) else torch.device(dev)
diff --git a/auto_round/compressors_new/base.py b/auto_round/compressors_new/base.py
index 8507e107e..09c01ed3f 100644
--- a/auto_round/compressors_new/base.py
+++ b/auto_round/compressors_new/base.py
@@ -279,9 +279,9 @@ class BaseCompressor(object):
# from quantize() as a fallback for non-AutoScheme cases.
self._post_init_done = False
- # Apply torch compile adjustments eagerly so that ar.enable_torch_compile
- # reflects the correct value immediately after construction (not only after post_init).
- self._adjust_torch_compile(enable_torch_compile)
+ # Apply eager state-only adjustments so ar.enable_torch_compile reflects
+ # unsupported combinations immediately after construction.
+ self._apply_eager_torch_compile_guards(enable_torch_compile)
self.compress_context.enable_torch_compile = self.enable_torch_compile
self.blocks_requiring_input_ids = []
@@ -571,11 +571,8 @@ class BaseCompressor(object):
def diffusion(self):
return self.model_context.is_diffusion
- def _adjust_torch_compile(self, enable_torch_compile: bool) -> None:
- """Sets the torch compile configuration for the tuning."""
- self.enable_torch_compile = enable_torch_compile
-
- # Determine fp8 / nvfp4 intent from raw config before scheme resolution.
+ def _get_torch_compile_state(self) -> dict[str, Any]:
+ """Collect raw torch.compile decision inputs from the current config state."""
cfg = self.quantize_config
raw_scheme = self.scheme if isinstance(self.scheme, str) else ""
raw_dt = (cfg.data_type or "").lower()
@@ -592,35 +589,54 @@ class BaseCompressor(object):
)
act_bits = getattr(cfg, "act_bits", 16) or 16
- if (
- not self.enable_torch_compile
- and TORCH_VERSION_AT_LEAST_2_6
- and act_bits > 8
- and not is_debug_mode()
- and not is_raw_fp8
- and self.need_calib
- ):
- logger.info(
- "%s",
- "'enable_torch_compile' is set to `False` by default. "
- "Enabling it can reduce tuning cost by 20%, but it might throw an exception.",
- )
+ super_group_size = getattr(cfg, "super_group_size", None)
+ enable_alg_ext = getattr(cfg, "enable_alg_ext", False)
+ return {
+ "is_raw_nv_fp": is_raw_nv_fp,
+ "is_raw_fp8": is_raw_fp8,
+ "act_bits": act_bits,
+ "super_group_size": super_group_size,
+ "enable_alg_ext": enable_alg_ext,
+ }
+
+ def _apply_eager_torch_compile_guards(self, enable_torch_compile: bool) -> None:
+ """Apply torch.compile disable rules that should take effect at construction time."""
+ self.enable_torch_compile = enable_torch_compile
+ state = self._get_torch_compile_state()
+
# On HPU, we rely on torch.compile to speed up the model execution.
- if self.enable_torch_compile and is_raw_fp8 and not is_hpex_available():
+ if self.enable_torch_compile and state["is_raw_fp8"] and not is_hpex_available():
self.enable_torch_compile = False
logger.warning_once("reset enable_torch_compile to `False` as fp8 is enabled")
# TODO: fix https://github.com/intel/auto-round/issues/1109
- if self.enable_torch_compile and is_raw_nv_fp:
+ if self.enable_torch_compile and state["is_raw_nv_fp"]:
self.enable_torch_compile = False
logger.warning_once("reset enable_torch_compile to `False` as nvfp4 is enabled")
- super_group_size = getattr(cfg, "super_group_size", None)
- enable_alg_ext = getattr(cfg, "enable_alg_ext", False)
- if self.enable_torch_compile and super_group_size is not None and enable_alg_ext:
+ if self.enable_torch_compile and state["super_group_size"] is not None and state["enable_alg_ext"]:
self.enable_torch_compile = False
logger.warning_once(
"reset enable_torch_compile to `False` as super_group_size is set for algorithm extension"
)
+ def _adjust_torch_compile(self, enable_torch_compile: bool) -> None:
+ """Finalize torch compile configuration for tuning after init state is resolved."""
+ state = self._get_torch_compile_state()
+ self._apply_eager_torch_compile_guards(enable_torch_compile)
+
+ if (
+ not enable_torch_compile
+ and TORCH_VERSION_AT_LEAST_2_6
+ and state["act_bits"] > 8
+ and not is_debug_mode()
+ and not state["is_raw_fp8"]
+ and self.need_calib
+ ):
+ logger.info(
+ "%s",
+ "'enable_torch_compile' is set to `False` by default. "
+ "Enabling it can reduce tuning cost by 20%, but it might throw an exception.",
+ )
+
def _get_calibration_dataset(self) -> str:
"""Resolve calibration dataset: self.dataset > AutoScheme.dataset > default."""
dataset = self.__dict__.get("dataset", None)
diff --git a/auto_round/compressors_new/entry.py b/auto_round/compressors_new/entry.py
index 1a9f984fd..c99f0977b 100644
--- a/auto_round/compressors_new/entry.py
+++ b/auto_round/compressors_new/entry.py
@@ -122,6 +122,19 @@ def is_weight_scheme(scheme):
return False
+def is_gguf_k_target(value) -> bool:
+ if isinstance(value, str):
+ normalized = value.strip().lower()
+ return normalized.startswith("gguf:") and "_k" in normalized
+ if isinstance(value, AutoScheme):
+ opts = value.options
+ if isinstance(opts, str):
+ opts = [opts]
+ if isinstance(opts, (list, tuple)):
+ return any(isinstance(opt, str) and is_gguf_k_target(opt) for opt in opts)
+ return False
+
+
def detect_model_type(model):
"""Detect the type of model (LLM, MLLM, or Diffusion).
@@ -227,6 +240,7 @@ class AutoRound(object):
elif isinstance(quant_config, RTNConfig):
enable_imatrix = False
+ _resolved = {}
disable_opt_rtn = getattr(quant_config, "disable_opt_rtn", False)
# If disable_opt_rtn was not explicitly set and scheme is W8A16/W8A8,
# auto-disable optimization to improve efficiency.
@@ -236,7 +250,7 @@ class AutoRound(object):
disable_opt_rtn = True
quant_config.disable_opt_rtn = True
if not disable_opt_rtn:
- has_gguf_k = "gguf" in format.lower() and "_k" in format.lower() if format else False
+ has_gguf_k = is_gguf_k_target(format) or is_gguf_k_target(scheme)
if has_gguf_k:
enable_imatrix = True
else:
diff --git a/auto_round/eval/evaluation.py b/auto_round/eval/evaluation.py
index e0af76396..9aad08223 100644
--- a/auto_round/eval/evaluation.py
+++ b/auto_round/eval/evaluation.py
@@ -21,6 +21,39 @@ from auto_round.utils import dispatch_model_block_wise
os.environ["TOKENIZERS_PARALLELISM"] = "false"
+def _collect_model_floating_dtypes(model) -> set:
+ floating_dtypes = set()
+ for parameter in model.parameters():
+ if parameter.is_floating_point():
+ floating_dtypes.add(parameter.dtype)
+ for buffer in model.buffers():
+ if buffer.is_floating_point():
+ floating_dtypes.add(buffer.dtype)
+ return floating_dtypes
+
+
+def _normalize_model_eval_dtype(model, eval_model_dtype):
+ import torch
+
+ floating_dtypes = _collect_model_floating_dtypes(model)
+ if not floating_dtypes:
+ return model
+
+ if eval_model_dtype == "auto":
+ if len(floating_dtypes) > 1:
+ target_dtype = torch.bfloat16 if torch.bfloat16 in floating_dtypes else getattr(model, "dtype", None)
+ if target_dtype is None or not getattr(target_dtype, "is_floating_point", False):
+ return model
+ logger.info("Normalizing mixed floating dtypes to %s for evaluation.", target_dtype)
+ return model.to(target_dtype)
+ return model
+
+ target_dtype = getattr(torch, eval_model_dtype)
+ if floating_dtypes != {target_dtype}:
+ return model.to(target_dtype)
+ return model
+
+
def simple_evaluate_user_model(
user_model,
tokenizer,
@@ -222,9 +255,7 @@ def prepare_model_for_eval(model, device_map, eval_model_dtype):
Returns:
model: Prepared model
"""
- import torch
-
- from auto_round.utils import detect_device
+ model = _normalize_model_eval_dtype(model, eval_model_dtype)
# Handle multi-device model
if hasattr(model, "hf_device_map") and len(model.hf_device_map) > 1:
@@ -234,10 +265,6 @@ def prepare_model_for_eval(model, device_map, eval_model_dtype):
else:
dispatch_model_block_wise(model, device_map)
- # Convert dtype
- if model.dtype != eval_model_dtype and eval_model_dtype != "auto":
- model.to(getattr(torch, eval_model_dtype))
-
return model
diff --git a/test/test_cpu/export/test_gguf_format.py b/test/test_cpu/export/test_gguf_format.py
index 9666bb286..e7c9a0584 100644
--- a/test/test_cpu/export/test_gguf_format.py
+++ b/test/test_cpu/export/test_gguf_format.py
@@ -53,6 +53,18 @@ class TestGGUF:
assert gguf_file.endswith(".gguf"), "Saved file is not in gguf format"
# Accuracy test is covered in test_cuda/export/test_gguf_format.py::TestAutoRound::test_q4_0_accuracy
+ def test_q2_k_s_routes_calibrated_rtn(self, tiny_qwen_model_path):
+ autoround = AutoRound(
+ tiny_qwen_model_path,
+ scheme="gguf:q2_k_s",
+ iters=0,
+ nsamples=1,
+ seqlen=8,
+ )
+
+ assert type(autoround).__name__ == "CalibratedRTNCompressor"
+ assert autoround.quantize_config._alg_cls == "OptimizedRTNQuantizer"
+
def test_func(self):
bits, group_size, sym = 4, 128, True
autoround = AutoRound(