Skip to content

PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度

fix gptqmodel inference issue (#813)

Wenhua Chengc4e7fbe5c5c5
diff --git a/auto_round/inference/backend.py b/auto_round/inference/backend.py
index 1868ee14c..8a946b966 100644
--- a/auto_round/inference/backend.py
+++ b/auto_round/inference/backend.py
@@ -559,8 +559,15 @@ def dynamic_import_inference_linear(backend, config):
def get_gptqmodel_infer_linear(backend, bits=4, group_size=128, sym=False):
+ import torch
+
+ dtype = torch.get_default_dtype()
+ if dtype != torch.float32:
+ torch.set_default_dtype(torch.float32)
import gptqmodel # pylint: disable=E0401
+ torch.set_default_dtype(dtype)
+
if "marlin" in backend:
return auto_round_extension.cuda.gptqmodel_marlin.get_marlin_layer()
# return gptqmodel.nn_modules.qlinear.marlin.MarlinQuantLinear