PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度
merge main branch to alg_ext (#970)
Wenhua Cheng9ecfaa5da578
diff --git a/auto_round/alg_ext.abi3.so b/auto_round/alg_ext.abi3.soindex 2f2907cfc..af9f2b29c 100755Binary files a/auto_round/alg_ext.abi3.so and b/auto_round/alg_ext.abi3.so differdiff --git a/auto_round/compressors/base.py b/auto_round/compressors/base.pyindex 602b566a2..c24537f8d 100644--- a/auto_round/compressors/base.py+++ b/auto_round/compressors/base.py@@ -2741,6 +2741,7 @@ class BaseCompressor(object):else:logger.info("using algorithm extension for quantization.")except (ImportError, ModuleNotFoundError):+ logger.error("algorithm extension import error, fallback to default mode")quantize_block = self._quantize_blockelse:quantize_block = self._quantize_blockdiff --git a/test/test_cpu/test_autoround.py b/test/test_cpu/test_autoround.pyindex 3adfd9f47..626dec380 100644--- a/test/test_cpu/test_autoround.py+++ b/test/test_cpu/test_autoround.py@@ -716,6 +716,9 @@ class TestAutoRound(unittest.TestCase):ar = AutoRound(model_name, scheme="W2A16", iters=1, nsamples=1, enable_alg_ext=True)ar.quantize()+ def test_alg_ext_import(self):+ from auto_round.alg_ext import quantize_block_ext+def test_invalid_layer_config(self):with self.assertRaises(ValueError):layer_config = {"model.decoder.layers.2.self_attnx": {"bits": 2}}diff --git a/test/test_cuda/test_alg_ext.py b/test/test_cuda/test_alg_ext.pynew file mode 100644index 000000000..06fcaf8a1--- /dev/null+++ b/test/test_cuda/test_alg_ext.py@@ -0,0 +1,40 @@+import shutil+import sys+import unittest++sys.path.insert(0, "../..")++import torch+from transformers import AutoModelForCausalLM, AutoTokenizer++from auto_round import AutoRound, AutoRoundConfig+from auto_round.eval.evaluation import simple_evaluate_user_model+++class TestAlgExt(unittest.TestCase):++ @classmethod+ def setUpClass(self):+ self.model_name = "/models/opt-125m"+ self.save_folder = "./saved"++ @classmethod+ def tearDownClass(self):+ shutil.rmtree(self.save_folder, ignore_errors=True)+ shutil.rmtree("runs", ignore_errors=True)++ def test_2bits(self):+ model_name = "/models/opt-125m"+ ar = AutoRound(model=model_name, bits=2, group_size=64, enable_alg_ext=True)+ ar.quantize_and_save(self.save_folder)+ model = AutoModelForCausalLM.from_pretrained(+ self.save_folder,+ device_map="auto",+ )++ tokenizer = AutoTokenizer.from_pretrained(self.save_folder)+ result = simple_evaluate_user_model(model, tokenizer, batch_size=64, tasks="lambada_openai")+ print(result["results"]["lambada_openai"]["acc,none"])+ # wo alg ext 0.2084, with 0.2364+ self.assertGreater(result["results"]["lambada_openai"]["acc,none"], 0.22)+ shutil.rmtree(self.save_folder, ignore_errors=True)