PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度
fix gptqmodel inference issue (#813)
Wenhua Chengc4e7fbe5c5c5
diff --git a/auto_round/inference/backend.py b/auto_round/inference/backend.pyindex 1868ee14c..8a946b966 100644--- a/auto_round/inference/backend.py+++ b/auto_round/inference/backend.py@@ -559,8 +559,15 @@ def dynamic_import_inference_linear(backend, config):def get_gptqmodel_infer_linear(backend, bits=4, group_size=128, sym=False):+ import torch++ dtype = torch.get_default_dtype()+ if dtype != torch.float32:+ torch.set_default_dtype(torch.float32)import gptqmodel # pylint: disable=E0401+ torch.set_default_dtype(dtype)+if "marlin" in backend:return auto_round_extension.cuda.gptqmodel_marlin.get_marlin_layer()# return gptqmodel.nn_modules.qlinear.marlin.MarlinQuantLinear