PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度
fix: preserve declared FP32 tensors during model dtype conversion (#2359)
Wang, Changb33eedc9bf99
diff --git a/auto_round/compressors/base.py b/auto_round/compressors/base.pyindex bda4df32d..4e1968684 100644--- a/auto_round/compressors/base.py+++ b/auto_round/compressors/base.py@@ -60,6 +60,7 @@ from auto_round.utils import (SUPPORTED_LAYER_TYPES,TORCH_VERSION_AT_LEAST_2_6,VISION_MM_KEYS,+ cast_model_dtype,compress_layer_names,convert_dtype_str2torch,extract_block_names_to_str,@@ -1400,7 +1401,7 @@ class BaseOrchestrator(object):logger.warning("force to use bf16 for quantization tuning when enabling activation quantization")self.model_context.amp_dtype = torch.bfloat16if self.model_context.model.dtype != torch.bfloat16:- self.model_context.model = self.model_context.model.to(torch.bfloat16)+ self.model_context.model = cast_model_dtype(self.model_context.model, torch.bfloat16)self._resolve_formats()self._patch_model()diff --git a/auto_round/compressors/diffusion_mixin.py b/auto_round/compressors/diffusion_mixin.pyindex 73e5a06e7..4f5ba2dd7 100644--- a/auto_round/compressors/diffusion_mixin.py+++ b/auto_round/compressors/diffusion_mixin.py@@ -25,7 +25,7 @@ from auto_round.utils.device import (get_major_device,)from auto_round.utils.device_manager import device_manager, is_auto_device_mapping-from auto_round.utils.model import rename_weights_files+from auto_round.utils.model import cast_model_dtype, rename_weights_filesclass DiffusionMixin:@@ -154,15 +154,7 @@ class DiffusionMixin:if not isinstance(component, torch.nn.Module):continue- fp32_modules = getattr(component, "_keep_in_fp32_modules", None) or []- tensors = list(component.named_parameters()) + list(component.named_buffers())- for tensor_name, tensor in tensors:- if not tensor.is_floating_point():- continue- keep_in_fp32 = any(module_name in tensor_name for module_name in fp32_modules)- desired_dtype = torch.float32 if keep_in_fp32 else target_dtype- if tensor.dtype != desired_dtype:- tensor.data = tensor.data.to(dtype=desired_dtype)+ cast_model_dtype(component, target_dtype)def _get_calibrator_kind(self) -> str:"""Select the diffusion calibration strategy.diff --git a/auto_round/context/model.py b/auto_round/context/model.pyindex 017b67ba5..eeffe50ae 100644--- a/auto_round/context/model.py+++ b/auto_round/context/model.py@@ -29,6 +29,7 @@ from auto_round.logger import loggerfrom auto_round.modeling.unfused_moe import apply_model_monkey_patchesfrom auto_round.special_model_handler import _handle_special_model, update_modulefrom auto_round.utils import (+ cast_model_dtype,check_and_mark_quantized_module,diffusion_load_model,install_debug_layer_config_patch,@@ -138,7 +139,7 @@ class ModelContext(BaseContext):logger.warning("force to use bf16 for quantization tuning when enabling activation quantization")self.amp_dtype = torch.bfloat16if self.model.dtype != torch.bfloat16:- self.model = self.model.to(torch.bfloat16)+ self.model = cast_model_dtype(self.model, torch.bfloat16)else:logger.debug(f"using {self.model.dtype} for quantization tuning")@@ -469,7 +470,7 @@ class ModelContext(BaseContext):)self.amp_dtype = amp_dtypeif self.model.dtype != self.amp_dtype:- self.model = self.model.to(self.amp_dtype)+ self.model = cast_model_dtype(self.model, self.amp_dtype)def apply_patches(self, formats):"""Apply format-specific model structure patches.@@ -493,7 +494,7 @@ class ModelContext(BaseContext):m.global_name = nif self.amp and self.model.dtype != self.amp_dtype:- self.model = self.model.to(self.amp_dtype)+ self.model = cast_model_dtype(self.model, self.amp_dtype)self._init_model = Trueself._is_initialized = Truediff --git a/auto_round/utils/model.py b/auto_round/utils/model.pyindex eb63f71f2..d7de6366f 100644--- a/auto_round/utils/model.py+++ b/auto_round/utils/model.py@@ -1789,17 +1789,45 @@ def check_seqlen_compatible(input_seqlen, tokenizer=None, model=None):)+def cast_model_dtype(model: torch.nn.Module, dtype: torch.dtype) -> torch.nn.Module:+ """Cast a model without rounding its declared FP32 parameters and buffers."""+ fp32_modules = set()+ for attribute in ("_keep_in_fp32_modules", "_keep_in_fp32_modules_strict"):+ names = getattr(model, attribute, None) or []+ fp32_modules.update([names] if isinstance(names, str) else names)+ if not fp32_modules:+ return model.to(dtype)++ # Inspect all aliases before casting so a shared tensor is protected even+ # when its first name is outside the FP32 modules.+ protected = set()+ tensors = list(model.named_parameters(remove_duplicate=False)) + list(model.named_buffers(remove_duplicate=False))+ for name, tensor in tensors:+ if any(module_name in name for module_name in fp32_modules):+ protected.add(id(tensor))+ if isinstance(tensor, torch.nn.Parameter) and tensor.grad is not None:+ protected.add(id(tensor.grad))++ def convert(tensor):+ if not (tensor.is_floating_point() or tensor.is_complex()):+ return tensor+ target_dtype = torch.float32 if id(tensor) in protected else dtype+ return tensor.to(dtype=target_dtype)++ return model._apply(convert)++def _to_model_dtype(model, model_dtype):- if model_dtype is not None:+ if isinstance(model_dtype, str):try:if (model_dtype == "float16" or model_dtype == "fp16") and model.dtype != torch.float16:- model = model.to(torch.float16)+ model = cast_model_dtype(model, torch.float16)elif (model_dtype == "bfloat16" or model_dtype == "bfp16" or model_dtype == "bf16") and model.dtype != torch.bfloat16:- model = model.to(torch.bfloat16)- elif model_dtype == "float32" or model_dtype == "fp32" and model.dtype != torch.bfloat32:- model = model.to(torch.float32)+ model = cast_model_dtype(model, torch.bfloat16)+ elif model_dtype == "float32" or model_dtype == "fp32":+ model = cast_model_dtype(model, torch.float32)except Exception:logger.error("please use more device to fit the device or just use one device")exit()diff --git a/test/unit/common/compressors/test_diffusion_mixin.py b/test/unit/common/compressors/test_diffusion_mixin.pyindex c05a51117..c16fb7a05 100644--- a/test/unit/common/compressors/test_diffusion_mixin.py+++ b/test/unit/common/compressors/test_diffusion_mixin.py@@ -15,6 +15,7 @@ import pytestimport torchfrom auto_round.compressors.diffusion_mixin import DiffusionMixin+from auto_round.context.model import ModelContextclass TestDiffusionMixinProperties:@@ -90,15 +91,27 @@ class TestDiffusionMixinProperties:comp.pipeline_call_kwargs = {"height": 512, "width": 512}assert comp.pipeline_call_kwargs.get("height") == 512- def test_align_pipeline_dtype_preserves_only_declared_fp32_tensors(self):+ def test_align_pipeline_dtype_preserves_only_declared_fp32_tensors(self, monkeypatch):protected = torch.nn.Linear(2, 2)protected._keep_in_fp32_modules = ["weight"]+ with torch.no_grad():+ protected.weight.fill_(1.001)+ expected_weight = protected.weight.detach().clone()ordinary = torch.nn.Linear(2, 2)pipe = SimpleNamespace(components=["protected", "ordinary"], protected=protected, ordinary=ordinary)+ # AMP runs before pipeline alignment; restoring FP32 afterwards is too late.+ protected.dtype = torch.float32+ context = SimpleNamespace(model=protected, amp=True, device="cpu")+ device = MagicMock()+ device.prefers_bf16.return_value = True+ device.supports_bf16.return_value = True+ monkeypatch.setattr("auto_round.context.model.get_ar_device", lambda _: device)+ ModelContext._set_amp_dtype(context)DiffusionMixin._align_pipeline_dtype(pipe, torch.bfloat16)assert protected.weight.dtype == torch.float32+ assert torch.equal(protected.weight, expected_weight)assert protected.bias.dtype == torch.bfloat16assert ordinary.weight.dtype == torch.bfloat16assert ordinary.bias.dtype == torch.bfloat16