Skip to content

PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度

Fix CI test_transformers issue caused by transformers 5.3.0 (#1535)

Liang Lv7b087fb25edf
diff --git a/auto_round/utils/common.py b/auto_round/utils/common.py
index decd5540f..126a23432 100644
--- a/auto_round/utils/common.py
+++ b/auto_round/utils/common.py
@@ -153,6 +153,34 @@ def _patch_transpose_for_buffers():
Transpose.convert = _patched_convert
+def _patch_tensor_get_dtype_for_prequantized_loading():
+ """Add a minimal ``torch.Tensor.get_dtype()`` shim for transformers 5.3.0.
+
+ transformers 5.3.0 added a pre-quantized loading branch in
+ ``core_model_loading.convert_and_load_state_dict_in_model()`` that calls
+ ``tensor.get_dtype()`` on checkpoint values. During actual weight loading,
+ those values are plain ``torch.Tensor`` instances returned by
+ ``modeling_utils.load_state_dict()``, which do not implement that method.
+
+ Older transformers versions only disabled dtype casting when a key was
+ renamed. The new version additionally inspects non-floating tensors, but it
+ does so through a safetensors-slice API that is not available on regular
+ tensors. Providing the method here preserves the new intent while restoring
+ compatibility for plain tensors.
+ """
+ if hasattr(torch.Tensor, "get_dtype"):
+ return
+
+ from safetensors.torch import _TYPES
+
+ torch_to_safetensors_dtype = {v: k for k, v in _TYPES.items()}
+
+ def _tensor_get_dtype(self):
+ return torch_to_safetensors_dtype.get(self.dtype, str(self.dtype).removeprefix("torch.").upper())
+
+ torch.Tensor.get_dtype = _tensor_get_dtype
+
+
# TODO: only AutoModelForCausalLM is patched; other Auto* classes are not covered yet
def monkey_patch_transformers():
transformers_version = getattr(transformers, "__version__", None)
@@ -177,6 +205,10 @@ def monkey_patch_transformers():
# transformers 5.2.0 added Transpose.convert() which calls get_parameter() on
# quantized buffer tensors (weight_packed, weight_scale), causing AttributeError.
_patch_transpose_for_buffers()
+ if parsed_version >= version.parse("5.3.0"):
+ # transformers 5.3.0 calls tensor.get_dtype() on plain torch.Tensor objects
+ # while loading pre-quantized checkpoints.
+ _patch_tensor_get_dtype_for_prequantized_loading()
if parsed_version >= version.parse("4.56.0"):
_patch_classmethod_kwargs(transformers.AutoModelForCausalLM, "from_pretrained", torch_dtype="dtype")
else: