PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度
Fix GGUF-K RTN routing and fake eval dtype normalization (#1807)
Heng Guo351ba32ad8ff
diff --git a/auto_round/__main__.py b/auto_round/__main__.pyindex a7e4a9569..42a230804 100644--- a/auto_round/__main__.py+++ b/auto_round/__main__.py@@ -806,6 +806,7 @@ def tune(args):autoround: BaseCompressor = AutoRound(model=model_name,platform=args.platform,+ format=args.format,scheme=scheme,dataset=args.dataset,iters=args.iters,diff --git a/auto_round/auto_scheme/delta_loss.py b/auto_round/auto_scheme/delta_loss.pyindex a7e2bdf90..792c71d10 100644--- a/auto_round/auto_scheme/delta_loss.py+++ b/auto_round/auto_scheme/delta_loss.py@@ -66,6 +66,7 @@ __all__ = ["gen_layer_config"]class AutoSchemeWrapperLinear(WrapperLinear):+def __init__(self,orig_layer,@@ -167,6 +168,7 @@ class AutoSchemeWrapperLinear(WrapperLinear):class AutoSchemeWrapperLinearForGGUFK(AutoSchemeWrapperLinear):+def __init__(self,orig_layer,@@ -212,6 +214,7 @@ class AutoSchemeWrapperLinearForGGUFK(AutoSchemeWrapperLinear):class AutoSchemeWrapperLinearForGGUFKImatrix(AutoSchemeWrapperLinear):+def __init__(self,orig_layer,@@ -291,6 +294,7 @@ class AutoSchemeWrapperLinearForGGUFKImatrix(AutoSchemeWrapperLinear):@torch.no_grad()def cal_imatrix(model, dataloader):+def register_act_hook(model):"""Registers hooks to accumulate activation squared norms into `imatrix`."""@@ -923,6 +927,7 @@ def choose_bits_per_layer_with_path(layers: dict, P: int):def move_module_to_tuning_device(module, major_device="cpu"):+def _normalize(dev):return dev if isinstance(dev, torch.device) else torch.device(dev)diff --git a/auto_round/compressors_new/base.py b/auto_round/compressors_new/base.pyindex 8507e107e..09c01ed3f 100644--- a/auto_round/compressors_new/base.py+++ b/auto_round/compressors_new/base.py@@ -279,9 +279,9 @@ class BaseCompressor(object):# from quantize() as a fallback for non-AutoScheme cases.self._post_init_done = False- # Apply torch compile adjustments eagerly so that ar.enable_torch_compile- # reflects the correct value immediately after construction (not only after post_init).- self._adjust_torch_compile(enable_torch_compile)+ # Apply eager state-only adjustments so ar.enable_torch_compile reflects+ # unsupported combinations immediately after construction.+ self._apply_eager_torch_compile_guards(enable_torch_compile)self.compress_context.enable_torch_compile = self.enable_torch_compileself.blocks_requiring_input_ids = []@@ -571,11 +571,8 @@ class BaseCompressor(object):def diffusion(self):return self.model_context.is_diffusion- def _adjust_torch_compile(self, enable_torch_compile: bool) -> None:- """Sets the torch compile configuration for the tuning."""- self.enable_torch_compile = enable_torch_compile-- # Determine fp8 / nvfp4 intent from raw config before scheme resolution.+ def _get_torch_compile_state(self) -> dict[str, Any]:+ """Collect raw torch.compile decision inputs from the current config state."""cfg = self.quantize_configraw_scheme = self.scheme if isinstance(self.scheme, str) else ""raw_dt = (cfg.data_type or "").lower()@@ -592,35 +589,54 @@ class BaseCompressor(object):)act_bits = getattr(cfg, "act_bits", 16) or 16- if (- not self.enable_torch_compile- and TORCH_VERSION_AT_LEAST_2_6- and act_bits > 8- and not is_debug_mode()- and not is_raw_fp8- and self.need_calib- ):- logger.info(- "%s",- "'enable_torch_compile' is set to `False` by default. "- "Enabling it can reduce tuning cost by 20%, but it might throw an exception.",- )+ super_group_size = getattr(cfg, "super_group_size", None)+ enable_alg_ext = getattr(cfg, "enable_alg_ext", False)+ return {+ "is_raw_nv_fp": is_raw_nv_fp,+ "is_raw_fp8": is_raw_fp8,+ "act_bits": act_bits,+ "super_group_size": super_group_size,+ "enable_alg_ext": enable_alg_ext,+ }++ def _apply_eager_torch_compile_guards(self, enable_torch_compile: bool) -> None:+ """Apply torch.compile disable rules that should take effect at construction time."""+ self.enable_torch_compile = enable_torch_compile+ state = self._get_torch_compile_state()+# On HPU, we rely on torch.compile to speed up the model execution.- if self.enable_torch_compile and is_raw_fp8 and not is_hpex_available():+ if self.enable_torch_compile and state["is_raw_fp8"] and not is_hpex_available():self.enable_torch_compile = Falselogger.warning_once("reset enable_torch_compile to `False` as fp8 is enabled")# TODO: fix https://github.com/intel/auto-round/issues/1109- if self.enable_torch_compile and is_raw_nv_fp:+ if self.enable_torch_compile and state["is_raw_nv_fp"]:self.enable_torch_compile = Falselogger.warning_once("reset enable_torch_compile to `False` as nvfp4 is enabled")- super_group_size = getattr(cfg, "super_group_size", None)- enable_alg_ext = getattr(cfg, "enable_alg_ext", False)- if self.enable_torch_compile and super_group_size is not None and enable_alg_ext:+ if self.enable_torch_compile and state["super_group_size"] is not None and state["enable_alg_ext"]:self.enable_torch_compile = Falselogger.warning_once("reset enable_torch_compile to `False` as super_group_size is set for algorithm extension")+ def _adjust_torch_compile(self, enable_torch_compile: bool) -> None:+ """Finalize torch compile configuration for tuning after init state is resolved."""+ state = self._get_torch_compile_state()+ self._apply_eager_torch_compile_guards(enable_torch_compile)++ if (+ not enable_torch_compile+ and TORCH_VERSION_AT_LEAST_2_6+ and state["act_bits"] > 8+ and not is_debug_mode()+ and not state["is_raw_fp8"]+ and self.need_calib+ ):+ logger.info(+ "%s",+ "'enable_torch_compile' is set to `False` by default. "+ "Enabling it can reduce tuning cost by 20%, but it might throw an exception.",+ )+def _get_calibration_dataset(self) -> str:"""Resolve calibration dataset: self.dataset > AutoScheme.dataset > default."""dataset = self.__dict__.get("dataset", None)diff --git a/auto_round/compressors_new/entry.py b/auto_round/compressors_new/entry.pyindex 1a9f984fd..c99f0977b 100644--- a/auto_round/compressors_new/entry.py+++ b/auto_round/compressors_new/entry.py@@ -122,6 +122,19 @@ def is_weight_scheme(scheme):return False+def is_gguf_k_target(value) -> bool:+ if isinstance(value, str):+ normalized = value.strip().lower()+ return normalized.startswith("gguf:") and "_k" in normalized+ if isinstance(value, AutoScheme):+ opts = value.options+ if isinstance(opts, str):+ opts = [opts]+ if isinstance(opts, (list, tuple)):+ return any(isinstance(opt, str) and is_gguf_k_target(opt) for opt in opts)+ return False++def detect_model_type(model):"""Detect the type of model (LLM, MLLM, or Diffusion).@@ -227,6 +240,7 @@ class AutoRound(object):elif isinstance(quant_config, RTNConfig):enable_imatrix = False+ _resolved = {}disable_opt_rtn = getattr(quant_config, "disable_opt_rtn", False)# If disable_opt_rtn was not explicitly set and scheme is W8A16/W8A8,# auto-disable optimization to improve efficiency.@@ -236,7 +250,7 @@ class AutoRound(object):disable_opt_rtn = Truequant_config.disable_opt_rtn = Trueif not disable_opt_rtn:- has_gguf_k = "gguf" in format.lower() and "_k" in format.lower() if format else False+ has_gguf_k = is_gguf_k_target(format) or is_gguf_k_target(scheme)if has_gguf_k:enable_imatrix = Trueelse:diff --git a/auto_round/eval/evaluation.py b/auto_round/eval/evaluation.pyindex e0af76396..9aad08223 100644--- a/auto_round/eval/evaluation.py+++ b/auto_round/eval/evaluation.py@@ -21,6 +21,39 @@ from auto_round.utils import dispatch_model_block_wiseos.environ["TOKENIZERS_PARALLELISM"] = "false"+def _collect_model_floating_dtypes(model) -> set:+ floating_dtypes = set()+ for parameter in model.parameters():+ if parameter.is_floating_point():+ floating_dtypes.add(parameter.dtype)+ for buffer in model.buffers():+ if buffer.is_floating_point():+ floating_dtypes.add(buffer.dtype)+ return floating_dtypes+++def _normalize_model_eval_dtype(model, eval_model_dtype):+ import torch++ floating_dtypes = _collect_model_floating_dtypes(model)+ if not floating_dtypes:+ return model++ if eval_model_dtype == "auto":+ if len(floating_dtypes) > 1:+ target_dtype = torch.bfloat16 if torch.bfloat16 in floating_dtypes else getattr(model, "dtype", None)+ if target_dtype is None or not getattr(target_dtype, "is_floating_point", False):+ return model+ logger.info("Normalizing mixed floating dtypes to %s for evaluation.", target_dtype)+ return model.to(target_dtype)+ return model++ target_dtype = getattr(torch, eval_model_dtype)+ if floating_dtypes != {target_dtype}:+ return model.to(target_dtype)+ return model++def simple_evaluate_user_model(user_model,tokenizer,@@ -222,9 +255,7 @@ def prepare_model_for_eval(model, device_map, eval_model_dtype):Returns:model: Prepared model"""- import torch-- from auto_round.utils import detect_device+ model = _normalize_model_eval_dtype(model, eval_model_dtype)# Handle multi-device modelif hasattr(model, "hf_device_map") and len(model.hf_device_map) > 1:@@ -234,10 +265,6 @@ def prepare_model_for_eval(model, device_map, eval_model_dtype):else:dispatch_model_block_wise(model, device_map)- # Convert dtype- if model.dtype != eval_model_dtype and eval_model_dtype != "auto":- model.to(getattr(torch, eval_model_dtype))-return modeldiff --git a/test/test_cpu/export/test_gguf_format.py b/test/test_cpu/export/test_gguf_format.pyindex 9666bb286..e7c9a0584 100644--- a/test/test_cpu/export/test_gguf_format.py+++ b/test/test_cpu/export/test_gguf_format.py@@ -53,6 +53,18 @@ class TestGGUF:assert gguf_file.endswith(".gguf"), "Saved file is not in gguf format"# Accuracy test is covered in test_cuda/export/test_gguf_format.py::TestAutoRound::test_q4_0_accuracy+ def test_q2_k_s_routes_calibrated_rtn(self, tiny_qwen_model_path):+ autoround = AutoRound(+ tiny_qwen_model_path,+ scheme="gguf:q2_k_s",+ iters=0,+ nsamples=1,+ seqlen=8,+ )++ assert type(autoround).__name__ == "CalibratedRTNCompressor"+ assert autoround.quantize_config._alg_cls == "OptimizedRTNQuantizer"+def test_func(self):bits, group_size, sym = 4, 128, Trueautoround = AutoRound(