Skip to content

PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度

Cast float32 weights to the export dtype when saving (#2385)

Shubham Padkonde0aa5e3515c41
diff --git a/auto_round/export/utils.py b/auto_round/export/utils.py
index 9037a234b..b834ea155 100644
--- a/auto_round/export/utils.py
+++ b/auto_round/export/utils.py
@@ -319,6 +319,35 @@ def _restore_original_layer_types(save_dir: str, source_dir: str) -> None:
json.dump(saved_config, f, indent=2)
+def _get_state_dict_for_export_dtype(model: nn.Module, dtype) -> dict | None:
+ """Return a state dict with float32 tensors cast to ``dtype``, or None if nothing needs casting.
+
+ Tuning may run the model in float32, e.g. when the device doesn't support bfloat16. The
+ exported config declares ``dtype``, so the saved tensors should use it too. Tensors of
+ modules the model keeps in float32 (``_keep_in_fp32_modules``) are left unchanged, as are
+ quantized (non-float32) tensors.
+ """
+ if dtype not in (torch.bfloat16, torch.float16):
+ return None
+ state_dict = model.state_dict()
+ if not any(tensor.dtype == torch.float32 for tensor in state_dict.values()):
+ return None
+
+ keep_in_fp32 = set()
+ for attribute in ("_keep_in_fp32_modules", "_keep_in_fp32_modules_strict"):
+ names = getattr(model, attribute, None) or []
+ keep_in_fp32.update([names] if isinstance(names, str) else names)
+
+ return {
+ name: (
+ tensor.to(dtype)
+ if tensor.dtype == torch.float32 and not any(module_name in name for module_name in keep_in_fp32)
+ else tensor
+ )
+ for name, tensor in state_dict.items()
+ }
+
+
def apply_post_save_source_fixes(model: nn.Module, save_dir: str) -> None:
"""Restore checkpoint artifacts that ``save_pretrained`` does not preserve."""
source_dir = _resolve_model_source_dir(model)
@@ -384,15 +413,19 @@ def save_model(
logger.info("Immediate saving mode: weights already saved by ShardWriter, saving configs only.")
_save_model_configs(model, save_dir)
else:
+ export_state_dict = _get_state_dict_for_export_dtype(model, dtype)
+ save_kwargs = {} if export_state_dict is None else {"state_dict": export_state_dict}
try:
- model.save_pretrained(save_dir, max_shard_size=max_shard_size, safe_serialization=safe_serialization)
+ model.save_pretrained(
+ save_dir, max_shard_size=max_shard_size, safe_serialization=safe_serialization, **save_kwargs
+ )
except (KeyError, TypeError) as e:
# Some third-party configs fail during config serialization in save_pretrained.
# Fall back to saving weights separately + config without diff.
logger.warning("model.save_pretrained failed (%s), falling back to manual save.", e)
from safetensors.torch import save_file
- state_dict = model.state_dict()
+ state_dict = model.state_dict() if export_state_dict is None else export_state_dict
save_file(state_dict, os.path.join(save_dir, "model.safetensors"))
_save_model_configs(model, save_dir)
diff --git a/test/unit/common/export/test_export_utils.py b/test/unit/common/export/test_export_utils.py
index 28a8c5ddf..7df0a6c9b 100644
--- a/test/unit/common/export/test_export_utils.py
+++ b/test/unit/common/export/test_export_utils.py
@@ -18,6 +18,7 @@ import torch
import torch.nn as nn
from auto_round.export.utils import (
+ _get_state_dict_for_export_dtype,
_resolve_model_source_dir,
_resolve_pipeline_source_dir,
_save_model_configs,
@@ -350,6 +351,66 @@ class TestSaveModel:
data = json.load(f)
assert data["torch_dtype"] == "bfloat16"
+ def test_dtype_casts_float32_weights_passed_to_save_pretrained(self):
+ with tempfile.TemporaryDirectory() as tmpdir:
+ model = MagicMock()
+ model.dtype = torch.float32
+ model.config.quantization_config = None
+ model.state_dict.return_value = {
+ "linear.weight": torch.ones(2, 2, dtype=torch.float32),
+ "linear.qweight": torch.ones(2, 2, dtype=torch.float8_e4m3fn),
+ }
+ model._keep_in_fp32_modules = None
+ model._keep_in_fp32_modules_strict = None
+ with patch("auto_round.export.utils._resolve_model_source_dir", return_value=None):
+ save_model(model, tmpdir, dtype=torch.bfloat16, safe_serialization=False)
+ state_dict = model.save_pretrained.call_args.kwargs["state_dict"]
+ assert state_dict["linear.weight"].dtype == torch.bfloat16
+ assert state_dict["linear.qweight"].dtype == torch.float8_e4m3fn
+
+
+class _ExportDtypeModel(nn.Module):
+ _keep_in_fp32_modules = ["router"]
+
+ def __init__(self):
+ super().__init__()
+ self.linear = nn.Linear(4, 4)
+ self.router = nn.Linear(4, 2)
+ self.register_buffer("qweight", torch.zeros(4, 4, dtype=torch.float8_e4m3fn))
+ self.register_buffer("indices", torch.zeros(4, dtype=torch.int32))
+
+
+class TestGetStateDictForExportDtype:
+ """Test _get_state_dict_for_export_dtype function."""
+
+ def test_casts_float32_tensors_to_export_dtype(self):
+ model = _ExportDtypeModel()
+ state_dict = _get_state_dict_for_export_dtype(model, torch.bfloat16)
+ assert state_dict["linear.weight"].dtype == torch.bfloat16
+ assert state_dict["linear.bias"].dtype == torch.bfloat16
+ # Quantized and integer tensors are unchanged.
+ assert state_dict["qweight"].dtype == torch.float8_e4m3fn
+ assert state_dict["indices"].dtype == torch.int32
+
+ def test_keeps_fp32_modules_in_float32(self):
+ model = _ExportDtypeModel()
+ state_dict = _get_state_dict_for_export_dtype(model, torch.float16)
+ assert state_dict["router.weight"].dtype == torch.float32
+ assert state_dict["linear.weight"].dtype == torch.float16
+
+ def test_does_not_modify_model(self):
+ model = _ExportDtypeModel()
+ _get_state_dict_for_export_dtype(model, torch.bfloat16)
+ assert model.linear.weight.dtype == torch.float32
+
+ @pytest.mark.parametrize("dtype", [None, torch.float32])
+ def test_returns_none_without_16bit_dtype(self, dtype):
+ assert _get_state_dict_for_export_dtype(_ExportDtypeModel(), dtype) is None
+
+ def test_returns_none_without_float32_tensors(self):
+ model = _ExportDtypeModel().to(torch.bfloat16)
+ assert _get_state_dict_for_export_dtype(model, torch.bfloat16) is None
+
# ==============================================================================
# get_autogptq_packing_qlinear