Skip to content

PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度

remove g_idx in gptq format (#267)

wenhuach21fdfd9711c5a7
diff --git a/README.md b/README.md
index 6f0c102de..c6cb4c581 100644
--- a/README.md
+++ b/README.md
@@ -27,7 +27,7 @@ more accuracy data and recipes across various models.
## What's New
* [2024/09] AutoRound format supports several LVM models, check out the examples [Qwen2-Vl](./examples/multimodal-modeling/Qwen-VL),[Phi-3-vision](./examples/multimodal-modeling/Phi-3-vision), [Llava](./examples/multimodal-modeling/Llava)
-* [2024/08] AutoRound format supports Intel Gaudi2 devices. For an example, please refer
+* [2024/08] AutoRound format supports Intel Gaudi2 devices. Please refer
to [Intel/Qwen2-7B-int4-inc](https://huggingface.co/Intel/Qwen2-7B-int4-inc).
* [2024/08] AutoRound introduces several experimental features, including fast tuning of norm/bias parameters (for 2-bit
and W4A4), activation quantization, and the mx_fp data type.
diff --git a/auto_round/export/export_to_autogptq/qlinear_triton.py b/auto_round/export/export_to_autogptq/qlinear_triton.py
index ea34c881a..cfb4a2a11 100644
--- a/auto_round/export/export_to_autogptq/qlinear_triton.py
+++ b/auto_round/export/export_to_autogptq/qlinear_triton.py
@@ -19,13 +19,13 @@ import torch
import torch.nn as nn
import transformers
+
class TritonModuleMixin:
@classmethod
def warmup(cls, model, transpose=False, seqlen=2048):
pass
-
class QuantLinear(nn.Module, TritonModuleMixin):
QUANT_TYPE = "triton"
@@ -39,7 +39,7 @@ class QuantLinear(nn.Module, TritonModuleMixin):
self.outfeatures = outfeatures
self.bits = bits
self.group_size = group_size if group_size != -1 else infeatures
- self.maxq = 2**self.bits - 1
+ self.maxq = 2 ** self.bits - 1
self.register_buffer(
"qweight",
@@ -96,7 +96,7 @@ class QuantLinear(nn.Module, TritonModuleMixin):
for idx in range(self.infeatures):
intweight.append(
torch.round((W[:, idx] + scale_zeros[self.g_idx[idx]]) / self.scales[self.g_idx[idx]]).to(torch.int)[
- :, None
+ :, None
]
)
intweight = torch.cat(intweight, dim=1)
@@ -135,8 +135,7 @@ class QuantLinear(nn.Module, TritonModuleMixin):
qzeros = qzeros.astype(np.int32)
self.qzeros = torch.from_numpy(qzeros)
+ delattr(self, "g_idx")
-
-
-__all__ = ["QuantLinear"]
\ No newline at end of file
+__all__ = ["QuantLinear"]
diff --git a/auto_round_extension/cuda/qlinear_exllamav2.py b/auto_round_extension/cuda/qlinear_exllamav2.py
index 3bc415971..786bb091d 100644
--- a/auto_round_extension/cuda/qlinear_exllamav2.py
+++ b/auto_round_extension/cuda/qlinear_exllamav2.py
@@ -196,10 +196,6 @@ class QuantLinear(nn.Module):
dtype=torch.float16,
),
)
- self.register_buffer(
- "g_idx",
- torch.tensor([i // self.group_size for i in range(infeatures)], dtype=torch.int32),
- )
if bias:
self.register_buffer("bias", torch.zeros((outfeatures), dtype=torch.float16))
@@ -214,7 +210,6 @@ class QuantLinear(nn.Module):
"qweight": self.qweight,
"qzeros": self.qzeros,
"scales": self.scales,
- "g_idx": self.g_idx,
}
temp_dq = temp_dq.get_scratch_slice(self.temp_dq_size())
self.q_handle = ext_make_q_matrix(self.q_tensors, temp_dq)