Skip to content

PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度

fix: preserve declared FP32 tensors during model dtype conversion (#2359)

Wang, Changb33eedc9bf99
diff --git a/auto_round/compressors/base.py b/auto_round/compressors/base.py
index bda4df32d..4e1968684 100644
--- a/auto_round/compressors/base.py
+++ b/auto_round/compressors/base.py
@@ -60,6 +60,7 @@ from auto_round.utils import (
SUPPORTED_LAYER_TYPES,
TORCH_VERSION_AT_LEAST_2_6,
VISION_MM_KEYS,
+ cast_model_dtype,
compress_layer_names,
convert_dtype_str2torch,
extract_block_names_to_str,
@@ -1400,7 +1401,7 @@ class BaseOrchestrator(object):
logger.warning("force to use bf16 for quantization tuning when enabling activation quantization")
self.model_context.amp_dtype = torch.bfloat16
if self.model_context.model.dtype != torch.bfloat16:
- self.model_context.model = self.model_context.model.to(torch.bfloat16)
+ self.model_context.model = cast_model_dtype(self.model_context.model, torch.bfloat16)
self._resolve_formats()
self._patch_model()
diff --git a/auto_round/compressors/diffusion_mixin.py b/auto_round/compressors/diffusion_mixin.py
index 73e5a06e7..4f5ba2dd7 100644
--- a/auto_round/compressors/diffusion_mixin.py
+++ b/auto_round/compressors/diffusion_mixin.py
@@ -25,7 +25,7 @@ from auto_round.utils.device import (
get_major_device,
)
from auto_round.utils.device_manager import device_manager, is_auto_device_mapping
-from auto_round.utils.model import rename_weights_files
+from auto_round.utils.model import cast_model_dtype, rename_weights_files
class DiffusionMixin:
@@ -154,15 +154,7 @@ class DiffusionMixin:
if not isinstance(component, torch.nn.Module):
continue
- fp32_modules = getattr(component, "_keep_in_fp32_modules", None) or []
- tensors = list(component.named_parameters()) + list(component.named_buffers())
- for tensor_name, tensor in tensors:
- if not tensor.is_floating_point():
- continue
- keep_in_fp32 = any(module_name in tensor_name for module_name in fp32_modules)
- desired_dtype = torch.float32 if keep_in_fp32 else target_dtype
- if tensor.dtype != desired_dtype:
- tensor.data = tensor.data.to(dtype=desired_dtype)
+ cast_model_dtype(component, target_dtype)
def _get_calibrator_kind(self) -> str:
"""Select the diffusion calibration strategy.
diff --git a/auto_round/context/model.py b/auto_round/context/model.py
index 017b67ba5..eeffe50ae 100644
--- a/auto_round/context/model.py
+++ b/auto_round/context/model.py
@@ -29,6 +29,7 @@ from auto_round.logger import logger
from auto_round.modeling.unfused_moe import apply_model_monkey_patches
from auto_round.special_model_handler import _handle_special_model, update_module
from auto_round.utils import (
+ cast_model_dtype,
check_and_mark_quantized_module,
diffusion_load_model,
install_debug_layer_config_patch,
@@ -138,7 +139,7 @@ class ModelContext(BaseContext):
logger.warning("force to use bf16 for quantization tuning when enabling activation quantization")
self.amp_dtype = torch.bfloat16
if self.model.dtype != torch.bfloat16:
- self.model = self.model.to(torch.bfloat16)
+ self.model = cast_model_dtype(self.model, torch.bfloat16)
else:
logger.debug(f"using {self.model.dtype} for quantization tuning")
@@ -469,7 +470,7 @@ class ModelContext(BaseContext):
)
self.amp_dtype = amp_dtype
if self.model.dtype != self.amp_dtype:
- self.model = self.model.to(self.amp_dtype)
+ self.model = cast_model_dtype(self.model, self.amp_dtype)
def apply_patches(self, formats):
"""Apply format-specific model structure patches.
@@ -493,7 +494,7 @@ class ModelContext(BaseContext):
m.global_name = n
if self.amp and self.model.dtype != self.amp_dtype:
- self.model = self.model.to(self.amp_dtype)
+ self.model = cast_model_dtype(self.model, self.amp_dtype)
self._init_model = True
self._is_initialized = True
diff --git a/auto_round/utils/model.py b/auto_round/utils/model.py
index eb63f71f2..d7de6366f 100644
--- a/auto_round/utils/model.py
+++ b/auto_round/utils/model.py
@@ -1789,17 +1789,45 @@ def check_seqlen_compatible(input_seqlen, tokenizer=None, model=None):
)
+def cast_model_dtype(model: torch.nn.Module, dtype: torch.dtype) -> torch.nn.Module:
+ """Cast a model without rounding its declared FP32 parameters and buffers."""
+ fp32_modules = set()
+ for attribute in ("_keep_in_fp32_modules", "_keep_in_fp32_modules_strict"):
+ names = getattr(model, attribute, None) or []
+ fp32_modules.update([names] if isinstance(names, str) else names)
+ if not fp32_modules:
+ return model.to(dtype)
+
+ # Inspect all aliases before casting so a shared tensor is protected even
+ # when its first name is outside the FP32 modules.
+ protected = set()
+ tensors = list(model.named_parameters(remove_duplicate=False)) + list(model.named_buffers(remove_duplicate=False))
+ for name, tensor in tensors:
+ if any(module_name in name for module_name in fp32_modules):
+ protected.add(id(tensor))
+ if isinstance(tensor, torch.nn.Parameter) and tensor.grad is not None:
+ protected.add(id(tensor.grad))
+
+ def convert(tensor):
+ if not (tensor.is_floating_point() or tensor.is_complex()):
+ return tensor
+ target_dtype = torch.float32 if id(tensor) in protected else dtype
+ return tensor.to(dtype=target_dtype)
+
+ return model._apply(convert)
+
+
def _to_model_dtype(model, model_dtype):
- if model_dtype is not None:
+ if isinstance(model_dtype, str):
try:
if (model_dtype == "float16" or model_dtype == "fp16") and model.dtype != torch.float16:
- model = model.to(torch.float16)
+ model = cast_model_dtype(model, torch.float16)
elif (
model_dtype == "bfloat16" or model_dtype == "bfp16" or model_dtype == "bf16"
) and model.dtype != torch.bfloat16:
- model = model.to(torch.bfloat16)
- elif model_dtype == "float32" or model_dtype == "fp32" and model.dtype != torch.bfloat32:
- model = model.to(torch.float32)
+ model = cast_model_dtype(model, torch.bfloat16)
+ elif model_dtype == "float32" or model_dtype == "fp32":
+ model = cast_model_dtype(model, torch.float32)
except Exception:
logger.error("please use more device to fit the device or just use one device")
exit()
diff --git a/test/unit/common/compressors/test_diffusion_mixin.py b/test/unit/common/compressors/test_diffusion_mixin.py
index c05a51117..c16fb7a05 100644
--- a/test/unit/common/compressors/test_diffusion_mixin.py
+++ b/test/unit/common/compressors/test_diffusion_mixin.py
@@ -15,6 +15,7 @@ import pytest
import torch
from auto_round.compressors.diffusion_mixin import DiffusionMixin
+from auto_round.context.model import ModelContext
class TestDiffusionMixinProperties:
@@ -90,15 +91,27 @@ class TestDiffusionMixinProperties:
comp.pipeline_call_kwargs = {"height": 512, "width": 512}
assert comp.pipeline_call_kwargs.get("height") == 512
- def test_align_pipeline_dtype_preserves_only_declared_fp32_tensors(self):
+ def test_align_pipeline_dtype_preserves_only_declared_fp32_tensors(self, monkeypatch):
protected = torch.nn.Linear(2, 2)
protected._keep_in_fp32_modules = ["weight"]
+ with torch.no_grad():
+ protected.weight.fill_(1.001)
+ expected_weight = protected.weight.detach().clone()
ordinary = torch.nn.Linear(2, 2)
pipe = SimpleNamespace(components=["protected", "ordinary"], protected=protected, ordinary=ordinary)
+ # AMP runs before pipeline alignment; restoring FP32 afterwards is too late.
+ protected.dtype = torch.float32
+ context = SimpleNamespace(model=protected, amp=True, device="cpu")
+ device = MagicMock()
+ device.prefers_bf16.return_value = True
+ device.supports_bf16.return_value = True
+ monkeypatch.setattr("auto_round.context.model.get_ar_device", lambda _: device)
+ ModelContext._set_amp_dtype(context)
DiffusionMixin._align_pipeline_dtype(pipe, torch.bfloat16)
assert protected.weight.dtype == torch.float32
+ assert torch.equal(protected.weight, expected_weight)
assert protected.bias.dtype == torch.bfloat16
assert ordinary.weight.dtype == torch.bfloat16
assert ordinary.bias.dtype == torch.bfloat16