Skip to content

PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度

fix rtn bug (#1622)

Wenhua Cheng891cf6ee51e4
diff --git a/auto_round/compressors/base.py b/auto_round/compressors/base.py
index 98b4389ce..998f964a1 100644
--- a/auto_round/compressors/base.py
+++ b/auto_round/compressors/base.py
@@ -1319,7 +1319,9 @@ class BaseCompressor(object):
tokenizer=self.tokenizer,
)
- @torch.inference_mode()
+ # Use no_grad instead of inference mode
+ # https://github.com/intel/auto-round/issues/1620
+ @torch.no_grad()
def _quantize_rtn(self) -> tuple[torch.nn.Module, dict[str, Any]]:
"""Quantize all modules in the model using RTN (Round-To-Nearest) strategy.
diff --git a/auto_round/utils/common.py b/auto_round/utils/common.py
index cfec1e300..4239980da 100644
--- a/auto_round/utils/common.py
+++ b/auto_round/utils/common.py
@@ -464,7 +464,7 @@ deepspeed_exists = False
if importlib.util.find_spec("deepspeed"): # check if deepspeed is installed
deepspeed_exists = True
-SUPPORTED_DTYPES = ("int", "mx_fp", "fp", "nv_fp")
+SUPPORTED_DTYPES = ("int", "mx_fp", "fp", "nv_fp", "mx_int")
SUPPORTED_FORMATS = SupportedFormats()
SUPPORTED_LAYER_TYPES = (torch.nn.Linear, transformers.pytorch_utils.Conv1D)
# Changed to str as it relies on triton or others lib to load this