PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度
fix cuda ut (#537)
Heng Guoeee1752bb1fe
diff --git a/auto_round/script/llm.py b/auto_round/script/llm.pyindex 5ec180548..b3685d9f6 100644--- a/auto_round/script/llm.py+++ b/auto_round/script/llm.py@@ -515,7 +515,7 @@ def tune(args):logger.info(f"Using lm-eval version {lm_eval_version}")eval_gguf_model = Falsefor file in os.listdir(eval_folder):- if file.endswith("guff"):+ if file.endswith("gguf"):eval_gguf_model = Truebreak@@ -595,7 +595,7 @@ def eval_task_by_task(from lm_eval.models.huggingface import HFLMfrom transformers import AutoModelForCausalLM, AutoTokenizer- # from auto_round import AutoRoundConfig+ from auto_round import AutoRoundConfig # pylint: disable=E0611if batch_size is None:batch_size = "auto"is_gguf_file = False@@ -604,7 +604,7 @@ def eval_task_by_task(else:if os.path.isfile(model) and model.endswith(".gguf"):is_gguf_file = True- gguf_file = model+ gguf_file = os.path.basename(model)model = os.path.dirname(model)else:for file in os.listdir(model):diff --git a/test/test_gguf_format.py b/test/test_gguf_format.pyindex ffa082c34..c85eded58 100644--- a/test/test_gguf_format.py+++ b/test/test_gguf_format.py@@ -36,12 +36,21 @@ class TestGGUF(unittest.TestCase):python_path = sys.executableres = os.system(f"cd .. && {python_path} -m auto_round --model {self.model_name} --eval_task_by_task"+ f" --tasks piqa --bs 16 --iters 1 --nsamples 1 --format fake,gguf:q4_0"+ )+ if res > 0 or res == -1:+ assert False, "cmd line test fail, please have a check"+ shutil.rmtree("./saved", ignore_errors=True)++ res = os.system(+ f"cd .. && {python_path} -m auto_round --model {self.model_name}"f" --tasks piqa,openbookqa --bs 16 --iters 1 --nsamples 1 --format fake,gguf:q4_0")if res > 0 or res == -1:assert False, "cmd line test fail, please have a check"shutil.rmtree("./saved", ignore_errors=True)+def test_q4_0(self):bits, group_size, sym = 4, 32, Trueautoround = AutoRound(diff --git a/test_cuda/test_auto_round_format.py b/test_cuda/test_auto_round_format.pyindex 6f8861b22..37f8c2a20 100644--- a/test_cuda/test_auto_round_format.py+++ b/test_cuda/test_auto_round_format.py@@ -3,9 +3,9 @@ import shutilimport sysimport unittest+sys.path.insert(0, "..")from auto_round.eval.evaluation import simple_evaluate_user_model-sys.path.insert(0, "..")import torchimport transformersfrom transformers import AutoModelForCausalLM, AutoTokenizerdiff --git a/test_cuda/test_exllamav2_backend.py b/test_cuda/test_exllamav2_backend.pyindex c62863873..66846c688 100644--- a/test_cuda/test_exllamav2_backend.py+++ b/test_cuda/test_exllamav2_backend.py@@ -2,14 +2,15 @@ import shutilimport sysimport unittestimport pytest-from auto_round.eval.evaluation import simple_evaluate_user_model-sys.path.insert(0, "..")++import torchfrom transformers import AutoModelForCausalLM, AutoTokenizerfrom auto_round import AutoRoundfrom auto_round import AutoRoundConfig+from auto_round.eval.evaluation import simple_evaluate_user_modelclass LLMDataLoader:diff --git a/test_cuda/test_marlin_backend.py b/test_cuda/test_marlin_backend.pyindex 3a231044f..a800eb00f 100644--- a/test_cuda/test_marlin_backend.py+++ b/test_cuda/test_marlin_backend.py@@ -2,7 +2,6 @@ import shutilimport sysimport unittestimport pytest-from auto_round.eval.evaluation import simple_evaluate_user_modelsys.path.insert(0, "..")import torch@@ -10,6 +9,7 @@ from transformers import AutoModelForCausalLM, AutoTokenizerfrom auto_round import AutoRoundfrom auto_round import AutoRoundConfig+from auto_round.eval.evaluation import simple_evaluate_user_modelclass LLMDataLoader:diff --git a/test_cuda/test_multiple_card.py b/test_cuda/test_multiple_card.pyindex 5a79d6647..2adffbaaa 100644--- a/test_cuda/test_multiple_card.py+++ b/test_cuda/test_multiple_card.py@@ -1,15 +1,15 @@-import shutil+import reimport sysimport unittest+import shutilsys.path.insert(0, "..")-from auto_round.eval.evaluation import simple_evaluate-from lm_eval.utils import make_table # pylint: disable=E0401-from auto_round import AutoRoundimport torch+from lm_eval.utils import make_table # pylint: disable=E0401from transformers import AutoModelForCausalLM, AutoTokenizer-import re+from auto_round import AutoRound+from auto_round.eval.evaluation import simple_evaluatedef get_accuracy(data):diff --git a/test_cuda/test_support_vlms.py b/test_cuda/test_support_vlms.pyindex b84836a80..4b76314d3 100644--- a/test_cuda/test_support_vlms.py+++ b/test_cuda/test_support_vlms.py@@ -325,7 +325,7 @@ class TestSupportVLMS(unittest.TestCase):shutil.rmtree(quantized_model_path, ignore_errors=True)def test_72b(self):- model_path = "/data5/models/Qwen2-VL-72B-Instruct/"+ model_path = "/models/Qwen2-VL-72B-Instruct/"res = os.system(f"cd .. && {self.python_path} -m auto_round --mllm "f"--model {model_path} --iter 1 --nsamples 1 --bs 1 --output_dir {self.save_dir} --device {self.device}"diff --git a/test_cuda/test_triton_backend.py b/test_cuda/test_triton_backend.pyindex 324d3ceaf..f87884f74 100644--- a/test_cuda/test_triton_backend.py+++ b/test_cuda/test_triton_backend.py@@ -3,9 +3,9 @@ import shutilimport sysimport unittest+sys.path.insert(0, "..")from auto_round.eval.evaluation import simple_evaluate_user_model-sys.path.insert(0, "..")import torchfrom transformers import AutoModelForCausalLM, AutoTokenizer@@ -135,7 +135,7 @@ class TestAutoRoundTritonBackend(unittest.TestCase):self.model_infer(model, tokenizer)result = simple_evaluate_user_model(model, tokenizer, batch_size=16, tasks="lambada_openai")print(result['results']['lambada_openai']['acc,none'])- self.assertGreater(result['results']['lambada_openai']['acc,none'], 0.20)+ self.assertGreater(result['results']['lambada_openai']['acc,none'], 0.19)torch.cuda.empty_cache()model = AutoModelForCausalLM.from_pretrained(@@ -149,7 +149,7 @@ class TestAutoRoundTritonBackend(unittest.TestCase):self.model_infer(model, tokenizer)result = simple_evaluate_user_model(model, tokenizer, batch_size=16, tasks="lambada_openai")print(result['results']['lambada_openai']['acc,none'])- self.assertGreater(result['results']['lambada_openai']['acc,none'], 0.20)+ self.assertGreater(result['results']['lambada_openai']['acc,none'], 0.19)torch.cuda.empty_cache()shutil.rmtree("./saved", ignore_errors=True)