PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度
fix rtn bug (#1622)
Wenhua Cheng891cf6ee51e4
diff --git a/auto_round/compressors/base.py b/auto_round/compressors/base.pyindex 98b4389ce..998f964a1 100644--- a/auto_round/compressors/base.py+++ b/auto_round/compressors/base.py@@ -1319,7 +1319,9 @@ class BaseCompressor(object):tokenizer=self.tokenizer,)- @torch.inference_mode()+ # Use no_grad instead of inference mode+ # https://github.com/intel/auto-round/issues/1620+ @torch.no_grad()def _quantize_rtn(self) -> tuple[torch.nn.Module, dict[str, Any]]:"""Quantize all modules in the model using RTN (Round-To-Nearest) strategy.diff --git a/auto_round/utils/common.py b/auto_round/utils/common.pyindex cfec1e300..4239980da 100644--- a/auto_round/utils/common.py+++ b/auto_round/utils/common.py@@ -464,7 +464,7 @@ deepspeed_exists = Falseif importlib.util.find_spec("deepspeed"): # check if deepspeed is installeddeepspeed_exists = True-SUPPORTED_DTYPES = ("int", "mx_fp", "fp", "nv_fp")+SUPPORTED_DTYPES = ("int", "mx_fp", "fp", "nv_fp", "mx_int")SUPPORTED_FORMATS = SupportedFormats()SUPPORTED_LAYER_TYPES = (torch.nn.Linear, transformers.pytorch_utils.Conv1D)# Changed to str as it relies on triton or others lib to load this