Skip to content

PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度

Preserve diffusion component dtypes during quantization (#2205)

Wang, Changd6ead479b21e
diff --git a/auto_round/compressors/diffusion_mixin.py b/auto_round/compressors/diffusion_mixin.py
index 9aa0a7b85..44465a244 100644
--- a/auto_round/compressors/diffusion_mixin.py
+++ b/auto_round/compressors/diffusion_mixin.py
@@ -120,7 +120,22 @@ class DiffusionMixin:
if pipe is not None and model is not None:
is_nextstep = hasattr(model, "config") and getattr(model.config, "model_type", None) == "nextstep"
if not is_nextstep:
- pipe.to(model.dtype)
+ self._align_pipeline_dtype(pipe, model.dtype)
+
+ @staticmethod
+ def _align_pipeline_dtype(pipe, target_dtype) -> None:
+ """Align ordinary components while preserving declared FP32 modules."""
+ for component_name in pipe.components:
+ component = getattr(pipe, component_name, None)
+ if (
+ not isinstance(component, torch.nn.Module)
+ or not hasattr(component, "dtype")
+ or component.dtype == target_dtype
+ ):
+ continue
+ if getattr(component, "_keep_in_fp32_modules", None):
+ continue
+ component.to(dtype=target_dtype)
def _get_calibrator_kind(self) -> str:
"""Select the diffusion calibration strategy.
@@ -150,16 +165,15 @@ class DiffusionMixin:
return result
def _align_device_and_dtype_for_secondary(self, transformer_name: str):
- """Align dtype and dispatch secondary transformer for multi-transformer pipelines."""
+ """Align safe component dtypes and dispatch a secondary transformer."""
pipe = getattr(self.model_context, "pipe", None)
model = getattr(self.model_context, "model", None)
if pipe is None or model is None:
return
- # Cast full pipeline to transformer's dtype
is_nextstep = hasattr(model, "config") and getattr(model.config, "model_type", None) == "nextstep"
if not is_nextstep:
- pipe.to(model.dtype)
+ self._align_pipeline_dtype(pipe, model.dtype)
# Dispatch secondary transformer to GPU(s)
device_map = getattr(self.compress_context, "device_map", None)
@@ -179,8 +193,6 @@ class DiffusionMixin:
)
is_other_component = not comp_name.startswith("transformer")
if is_other_transformer or is_other_component:
- if isinstance(comp, torch.nn.Module) and hasattr(comp, "dtype") and comp.dtype != model.dtype:
- comp.to(dtype=model.dtype)
try:
comp.to(comp_device)
except (NotImplementedError, RuntimeError):
@@ -435,7 +447,7 @@ class DiffusionMixin:
self.model_context.quantized = False
self._post_init_done = False
- # Re-align device/dtype and dispatch pipeline for secondary transformer
+ # Dispatch the pipeline for the secondary transformer without recasting it.
self._align_device_and_dtype_for_secondary(comp_name)
# Re-run post_init to set up quantizer for new model
diff --git a/test/unit/test_cpu/compressors/test_diffusion_mixin.py b/test/unit/test_cpu/compressors/test_diffusion_mixin.py
index 49e89eb0a..648e8eab0 100644
--- a/test/unit/test_cpu/compressors/test_diffusion_mixin.py
+++ b/test/unit/test_cpu/compressors/test_diffusion_mixin.py
@@ -47,6 +47,19 @@ class TestDiffusionMixinProperties:
comp.pipeline_call_kwargs = {"height": 512, "width": 512}
assert comp.pipeline_call_kwargs.get("height") == 512
+ def test_align_pipeline_dtype_preserves_fp32_modules(self):
+ protected = torch.nn.Linear(2, 2)
+ protected._keep_in_fp32_modules = ["weight"]
+ protected.dtype = torch.float32
+ ordinary = torch.nn.Linear(2, 2)
+ ordinary.dtype = torch.float32
+ pipe = SimpleNamespace(components=["protected", "ordinary"], protected=protected, ordinary=ordinary)
+
+ DiffusionMixin._align_pipeline_dtype(pipe, torch.bfloat16)
+
+ assert protected.weight.dtype == torch.float32
+ assert ordinary.weight.dtype == torch.bfloat16
+
class TestFindAdditionalTransformers:
"""Test _find_additional_transformers logic."""