PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度
Preserve diffusion component dtypes during quantization (#2205)
Wang, Changd6ead479b21e
diff --git a/auto_round/compressors/diffusion_mixin.py b/auto_round/compressors/diffusion_mixin.pyindex 9aa0a7b85..44465a244 100644--- a/auto_round/compressors/diffusion_mixin.py+++ b/auto_round/compressors/diffusion_mixin.py@@ -120,7 +120,22 @@ class DiffusionMixin:if pipe is not None and model is not None:is_nextstep = hasattr(model, "config") and getattr(model.config, "model_type", None) == "nextstep"if not is_nextstep:- pipe.to(model.dtype)+ self._align_pipeline_dtype(pipe, model.dtype)++ @staticmethod+ def _align_pipeline_dtype(pipe, target_dtype) -> None:+ """Align ordinary components while preserving declared FP32 modules."""+ for component_name in pipe.components:+ component = getattr(pipe, component_name, None)+ if (+ not isinstance(component, torch.nn.Module)+ or not hasattr(component, "dtype")+ or component.dtype == target_dtype+ ):+ continue+ if getattr(component, "_keep_in_fp32_modules", None):+ continue+ component.to(dtype=target_dtype)def _get_calibrator_kind(self) -> str:"""Select the diffusion calibration strategy.@@ -150,16 +165,15 @@ class DiffusionMixin:return resultdef _align_device_and_dtype_for_secondary(self, transformer_name: str):- """Align dtype and dispatch secondary transformer for multi-transformer pipelines."""+ """Align safe component dtypes and dispatch a secondary transformer."""pipe = getattr(self.model_context, "pipe", None)model = getattr(self.model_context, "model", None)if pipe is None or model is None:return- # Cast full pipeline to transformer's dtypeis_nextstep = hasattr(model, "config") and getattr(model.config, "model_type", None) == "nextstep"if not is_nextstep:- pipe.to(model.dtype)+ self._align_pipeline_dtype(pipe, model.dtype)# Dispatch secondary transformer to GPU(s)device_map = getattr(self.compress_context, "device_map", None)@@ -179,8 +193,6 @@ class DiffusionMixin:)is_other_component = not comp_name.startswith("transformer")if is_other_transformer or is_other_component:- if isinstance(comp, torch.nn.Module) and hasattr(comp, "dtype") and comp.dtype != model.dtype:- comp.to(dtype=model.dtype)try:comp.to(comp_device)except (NotImplementedError, RuntimeError):@@ -435,7 +447,7 @@ class DiffusionMixin:self.model_context.quantized = Falseself._post_init_done = False- # Re-align device/dtype and dispatch pipeline for secondary transformer+ # Dispatch the pipeline for the secondary transformer without recasting it.self._align_device_and_dtype_for_secondary(comp_name)# Re-run post_init to set up quantizer for new modeldiff --git a/test/unit/test_cpu/compressors/test_diffusion_mixin.py b/test/unit/test_cpu/compressors/test_diffusion_mixin.pyindex 49e89eb0a..648e8eab0 100644--- a/test/unit/test_cpu/compressors/test_diffusion_mixin.py+++ b/test/unit/test_cpu/compressors/test_diffusion_mixin.py@@ -47,6 +47,19 @@ class TestDiffusionMixinProperties:comp.pipeline_call_kwargs = {"height": 512, "width": 512}assert comp.pipeline_call_kwargs.get("height") == 512+ def test_align_pipeline_dtype_preserves_fp32_modules(self):+ protected = torch.nn.Linear(2, 2)+ protected._keep_in_fp32_modules = ["weight"]+ protected.dtype = torch.float32+ ordinary = torch.nn.Linear(2, 2)+ ordinary.dtype = torch.float32+ pipe = SimpleNamespace(components=["protected", "ordinary"], protected=protected, ordinary=ordinary)++ DiffusionMixin._align_pipeline_dtype(pipe, torch.bfloat16)++ assert protected.weight.dtype == torch.float32+ assert ordinary.weight.dtype == torch.bfloat16+class TestFindAdditionalTransformers:"""Test _find_additional_transformers logic."""