Skip to content

PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度

fix bug of q4_0 and q5_0 at iters==0 (#658)

Heng Guo750a5c10f731
diff --git a/auto_round/export/export_to_gguf/packing.py b/auto_round/export/export_to_gguf/packing.py
index 98a5223fe..80c2b1abd 100644
--- a/auto_round/export/export_to_gguf/packing.py
+++ b/auto_round/export/export_to_gguf/packing.py
@@ -300,7 +300,7 @@ def q4_0_quant_block(blocks, scale=None, zp=None, **kwargs):
d = scale.reshape((-1, 1))
else:
imax = abs(blocks).argmax(axis=-1, keepdims=True)
- max = np.take_along_axis(blocks, imax, axis=-1)
+ max = torch.take_along_dim(blocks, imax, axis=-1)
d = max / -8
id = torch.where(d == 0, 0, 1 / d)
@@ -345,7 +345,7 @@ def q5_0_quant_block(blocks: np.array, scale=None, zp=None, **kwargs):
d = scale.reshape((-1, 1))
else:
imax = abs(blocks).argmax(axis=-1, keepdims=True)
- max = torch.take_along_dim(blocks, imax, dim=-1)[0]
+ max = torch.take_along_dim(blocks, imax, dim=-1)
d = max / -16
id = torch.where(d == 0, 0, 1 / d)
diff --git a/auto_round/wrapper.py b/auto_round/wrapper.py
index 543392c62..98f9599f7 100644
--- a/auto_round/wrapper.py
+++ b/auto_round/wrapper.py
@@ -240,8 +240,8 @@ class WrapperLinear(torch.nn.Module):
self.orig_layer.to(self.device)
##unwrapper weight
qdq_weight, scale, zp = self._qdq_weight(v, min_scale, max_scale)
- if hasattr(self.orig_layer, "imatrix"):
- self.orig_layer.imatrix = None
+ # if hasattr(self.orig_layer, "imatrix"):
+ # self.orig_layer.imatrix = None
self.orig_layer.weight.data.copy_(qdq_weight)
self.orig_layer.weight.grad = None
diff --git a/test/test_cpu/test_gguf_format.py b/test/test_cpu/test_gguf_format.py
index 6a5192fdd..4f33b7204 100644
--- a/test/test_cpu/test_gguf_format.py
+++ b/test/test_cpu/test_gguf_format.py
@@ -269,6 +269,27 @@ class TestGGUF(unittest.TestCase):
self.assertEqual(autoround.model.model.layers[10].mlp.gate_proj.bits, 8)
self.assertEqual(autoround.layer_config['model.layers.10.mlp.gate_proj']['mostly'], "gguf:q8_0")
shutil.rmtree("./saved", ignore_errors=True)
+
+ def test_all_format(self):
+ from auto_round.export.export_to_gguf.config import GGUF_CONFIG
+ model_name = "Qwen/Qwen2.5-7B-Instruct"
+ python_path = sys.executable
+ for gguf_format in GGUF_CONFIG.keys():
+ res = os.system(
+ f"cd ../.. && {python_path} -m auto_round --model {model_name} "
+ f" --bs 16 --iters 1 --nsamples 1 --format {gguf_format}"
+ )
+ if res > 0 or res == -1:
+ assert False, "cmd line test fail, please have a check"
+ shutil.rmtree("../../tmp_autoround", ignore_errors=True)
+
+ res = os.system(
+ f"cd ../.. && {python_path} -m auto_round --model {model_name} "
+ f" --bs 16 --iters 0 --nsamples 1 --format fake,{gguf_format}"
+ )
+ if res > 0 or res == -1:
+ assert False, "cmd line test fail, please have a check"
+ shutil.rmtree("../../tmp_autoround", ignore_errors=True)
if __name__ == "__main__":
unittest.main()
diff --git a/test/test_cuda/test_gguf.py b/test/test_cuda/test_gguf.py
index 405596c89..05e212e7e 100644
--- a/test/test_cuda/test_gguf.py
+++ b/test/test_cuda/test_gguf.py
@@ -171,5 +171,26 @@ class TestAutoRound(unittest.TestCase):
self.assertGreater(result['results']['piqa']['acc,none'], 0.55)
shutil.rmtree("./saved", ignore_errors=True)
+ def test_all_format(self):
+ from auto_round.export.export_to_gguf.config import GGUF_CONFIG
+ python_path = sys.executable
+ for model_name in ["/models/Qwen3-8B/", "/models/Llama-3.2-3B/"]:
+ for gguf_format in GGUF_CONFIG.keys():
+ res = os.system(
+ f"cd ../.. && {python_path} -m auto_round --model {model_name} "
+ f" --bs 16 --iters 1 --nsamples 1 --format fake,{gguf_format}"
+ )
+ if res > 0 or res == -1:
+ assert False, "cmd line test fail, please have a check"
+ shutil.rmtree("../../tmp_autoround", ignore_errors=True)
+
+ res = os.system(
+ f"cd ../.. && {python_path} -m auto_round --model {model_name} "
+ f" --bs 16 --iters 0 --nsamples 1 --format {gguf_format}"
+ )
+ if res > 0 or res == -1:
+ assert False, "cmd line test fail, please have a check"
+ shutil.rmtree("../../tmp_autoround", ignore_errors=True)
+
if __name__ == "__main__":
unittest.main()
\ No newline at end of file