PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度
Cast float32 weights to the export dtype when saving (#2385)
Shubham Padkonde0aa5e3515c41
diff --git a/auto_round/export/utils.py b/auto_round/export/utils.pyindex 9037a234b..b834ea155 100644--- a/auto_round/export/utils.py+++ b/auto_round/export/utils.py@@ -319,6 +319,35 @@ def _restore_original_layer_types(save_dir: str, source_dir: str) -> None:json.dump(saved_config, f, indent=2)+def _get_state_dict_for_export_dtype(model: nn.Module, dtype) -> dict | None:+ """Return a state dict with float32 tensors cast to ``dtype``, or None if nothing needs casting.++ Tuning may run the model in float32, e.g. when the device doesn't support bfloat16. The+ exported config declares ``dtype``, so the saved tensors should use it too. Tensors of+ modules the model keeps in float32 (``_keep_in_fp32_modules``) are left unchanged, as are+ quantized (non-float32) tensors.+ """+ if dtype not in (torch.bfloat16, torch.float16):+ return None+ state_dict = model.state_dict()+ if not any(tensor.dtype == torch.float32 for tensor in state_dict.values()):+ return None++ keep_in_fp32 = set()+ for attribute in ("_keep_in_fp32_modules", "_keep_in_fp32_modules_strict"):+ names = getattr(model, attribute, None) or []+ keep_in_fp32.update([names] if isinstance(names, str) else names)++ return {+ name: (+ tensor.to(dtype)+ if tensor.dtype == torch.float32 and not any(module_name in name for module_name in keep_in_fp32)+ else tensor+ )+ for name, tensor in state_dict.items()+ }++def apply_post_save_source_fixes(model: nn.Module, save_dir: str) -> None:"""Restore checkpoint artifacts that ``save_pretrained`` does not preserve."""source_dir = _resolve_model_source_dir(model)@@ -384,15 +413,19 @@ def save_model(logger.info("Immediate saving mode: weights already saved by ShardWriter, saving configs only.")_save_model_configs(model, save_dir)else:+ export_state_dict = _get_state_dict_for_export_dtype(model, dtype)+ save_kwargs = {} if export_state_dict is None else {"state_dict": export_state_dict}try:- model.save_pretrained(save_dir, max_shard_size=max_shard_size, safe_serialization=safe_serialization)+ model.save_pretrained(+ save_dir, max_shard_size=max_shard_size, safe_serialization=safe_serialization, **save_kwargs+ )except (KeyError, TypeError) as e:# Some third-party configs fail during config serialization in save_pretrained.# Fall back to saving weights separately + config without diff.logger.warning("model.save_pretrained failed (%s), falling back to manual save.", e)from safetensors.torch import save_file- state_dict = model.state_dict()+ state_dict = model.state_dict() if export_state_dict is None else export_state_dictsave_file(state_dict, os.path.join(save_dir, "model.safetensors"))_save_model_configs(model, save_dir)diff --git a/test/unit/common/export/test_export_utils.py b/test/unit/common/export/test_export_utils.pyindex 28a8c5ddf..7df0a6c9b 100644--- a/test/unit/common/export/test_export_utils.py+++ b/test/unit/common/export/test_export_utils.py@@ -18,6 +18,7 @@ import torchimport torch.nn as nnfrom auto_round.export.utils import (+ _get_state_dict_for_export_dtype,_resolve_model_source_dir,_resolve_pipeline_source_dir,_save_model_configs,@@ -350,6 +351,66 @@ class TestSaveModel:data = json.load(f)assert data["torch_dtype"] == "bfloat16"+ def test_dtype_casts_float32_weights_passed_to_save_pretrained(self):+ with tempfile.TemporaryDirectory() as tmpdir:+ model = MagicMock()+ model.dtype = torch.float32+ model.config.quantization_config = None+ model.state_dict.return_value = {+ "linear.weight": torch.ones(2, 2, dtype=torch.float32),+ "linear.qweight": torch.ones(2, 2, dtype=torch.float8_e4m3fn),+ }+ model._keep_in_fp32_modules = None+ model._keep_in_fp32_modules_strict = None+ with patch("auto_round.export.utils._resolve_model_source_dir", return_value=None):+ save_model(model, tmpdir, dtype=torch.bfloat16, safe_serialization=False)+ state_dict = model.save_pretrained.call_args.kwargs["state_dict"]+ assert state_dict["linear.weight"].dtype == torch.bfloat16+ assert state_dict["linear.qweight"].dtype == torch.float8_e4m3fn+++class _ExportDtypeModel(nn.Module):+ _keep_in_fp32_modules = ["router"]++ def __init__(self):+ super().__init__()+ self.linear = nn.Linear(4, 4)+ self.router = nn.Linear(4, 2)+ self.register_buffer("qweight", torch.zeros(4, 4, dtype=torch.float8_e4m3fn))+ self.register_buffer("indices", torch.zeros(4, dtype=torch.int32))+++class TestGetStateDictForExportDtype:+ """Test _get_state_dict_for_export_dtype function."""++ def test_casts_float32_tensors_to_export_dtype(self):+ model = _ExportDtypeModel()+ state_dict = _get_state_dict_for_export_dtype(model, torch.bfloat16)+ assert state_dict["linear.weight"].dtype == torch.bfloat16+ assert state_dict["linear.bias"].dtype == torch.bfloat16+ # Quantized and integer tensors are unchanged.+ assert state_dict["qweight"].dtype == torch.float8_e4m3fn+ assert state_dict["indices"].dtype == torch.int32++ def test_keeps_fp32_modules_in_float32(self):+ model = _ExportDtypeModel()+ state_dict = _get_state_dict_for_export_dtype(model, torch.float16)+ assert state_dict["router.weight"].dtype == torch.float32+ assert state_dict["linear.weight"].dtype == torch.float16++ def test_does_not_modify_model(self):+ model = _ExportDtypeModel()+ _get_state_dict_for_export_dtype(model, torch.bfloat16)+ assert model.linear.weight.dtype == torch.float32++ @pytest.mark.parametrize("dtype", [None, torch.float32])+ def test_returns_none_without_16bit_dtype(self, dtype):+ assert _get_state_dict_for_export_dtype(_ExportDtypeModel(), dtype) is None++ def test_returns_none_without_float32_tensors(self):+ model = _ExportDtypeModel().to(torch.bfloat16)+ assert _get_state_dict_for_export_dtype(model, torch.bfloat16) is None+# ==============================================================================# get_autogptq_packing_qlinear