PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度
fix bug of q4_0 and q5_0 at iters==0 (#658)
Heng Guo750a5c10f731
diff --git a/auto_round/export/export_to_gguf/packing.py b/auto_round/export/export_to_gguf/packing.pyindex 98a5223fe..80c2b1abd 100644--- a/auto_round/export/export_to_gguf/packing.py+++ b/auto_round/export/export_to_gguf/packing.py@@ -300,7 +300,7 @@ def q4_0_quant_block(blocks, scale=None, zp=None, **kwargs):d = scale.reshape((-1, 1))else:imax = abs(blocks).argmax(axis=-1, keepdims=True)- max = np.take_along_axis(blocks, imax, axis=-1)+ max = torch.take_along_dim(blocks, imax, axis=-1)d = max / -8id = torch.where(d == 0, 0, 1 / d)@@ -345,7 +345,7 @@ def q5_0_quant_block(blocks: np.array, scale=None, zp=None, **kwargs):d = scale.reshape((-1, 1))else:imax = abs(blocks).argmax(axis=-1, keepdims=True)- max = torch.take_along_dim(blocks, imax, dim=-1)[0]+ max = torch.take_along_dim(blocks, imax, dim=-1)d = max / -16id = torch.where(d == 0, 0, 1 / d)diff --git a/auto_round/wrapper.py b/auto_round/wrapper.pyindex 543392c62..98f9599f7 100644--- a/auto_round/wrapper.py+++ b/auto_round/wrapper.py@@ -240,8 +240,8 @@ class WrapperLinear(torch.nn.Module):self.orig_layer.to(self.device)##unwrapper weightqdq_weight, scale, zp = self._qdq_weight(v, min_scale, max_scale)- if hasattr(self.orig_layer, "imatrix"):- self.orig_layer.imatrix = None+ # if hasattr(self.orig_layer, "imatrix"):+ # self.orig_layer.imatrix = Noneself.orig_layer.weight.data.copy_(qdq_weight)self.orig_layer.weight.grad = Nonediff --git a/test/test_cpu/test_gguf_format.py b/test/test_cpu/test_gguf_format.pyindex 6a5192fdd..4f33b7204 100644--- a/test/test_cpu/test_gguf_format.py+++ b/test/test_cpu/test_gguf_format.py@@ -269,6 +269,27 @@ class TestGGUF(unittest.TestCase):self.assertEqual(autoround.model.model.layers[10].mlp.gate_proj.bits, 8)self.assertEqual(autoround.layer_config['model.layers.10.mlp.gate_proj']['mostly'], "gguf:q8_0")shutil.rmtree("./saved", ignore_errors=True)++ def test_all_format(self):+ from auto_round.export.export_to_gguf.config import GGUF_CONFIG+ model_name = "Qwen/Qwen2.5-7B-Instruct"+ python_path = sys.executable+ for gguf_format in GGUF_CONFIG.keys():+ res = os.system(+ f"cd ../.. && {python_path} -m auto_round --model {model_name} "+ f" --bs 16 --iters 1 --nsamples 1 --format {gguf_format}"+ )+ if res > 0 or res == -1:+ assert False, "cmd line test fail, please have a check"+ shutil.rmtree("../../tmp_autoround", ignore_errors=True)++ res = os.system(+ f"cd ../.. && {python_path} -m auto_round --model {model_name} "+ f" --bs 16 --iters 0 --nsamples 1 --format fake,{gguf_format}"+ )+ if res > 0 or res == -1:+ assert False, "cmd line test fail, please have a check"+ shutil.rmtree("../../tmp_autoround", ignore_errors=True)if __name__ == "__main__":unittest.main()diff --git a/test/test_cuda/test_gguf.py b/test/test_cuda/test_gguf.pyindex 405596c89..05e212e7e 100644--- a/test/test_cuda/test_gguf.py+++ b/test/test_cuda/test_gguf.py@@ -171,5 +171,26 @@ class TestAutoRound(unittest.TestCase):self.assertGreater(result['results']['piqa']['acc,none'], 0.55)shutil.rmtree("./saved", ignore_errors=True)+ def test_all_format(self):+ from auto_round.export.export_to_gguf.config import GGUF_CONFIG+ python_path = sys.executable+ for model_name in ["/models/Qwen3-8B/", "/models/Llama-3.2-3B/"]:+ for gguf_format in GGUF_CONFIG.keys():+ res = os.system(+ f"cd ../.. && {python_path} -m auto_round --model {model_name} "+ f" --bs 16 --iters 1 --nsamples 1 --format fake,{gguf_format}"+ )+ if res > 0 or res == -1:+ assert False, "cmd line test fail, please have a check"+ shutil.rmtree("../../tmp_autoround", ignore_errors=True)++ res = os.system(+ f"cd ../.. && {python_path} -m auto_round --model {model_name} "+ f" --bs 16 --iters 0 --nsamples 1 --format {gguf_format}"+ )+ if res > 0 or res == -1:+ assert False, "cmd line test fail, please have a check"+ shutil.rmtree("../../tmp_autoround", ignore_errors=True)+if __name__ == "__main__":unittest.main()\ No newline at end of file