PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度
Fix torch_zp infer bug & API disable_deterministic_algorithms bug (#733)
Weiwei71dab1dca142
diff --git a/auto_round/autoround.py b/auto_round/autoround.pyindex b143cac36..3c0b07247 100644--- a/auto_round/autoround.py+++ b/auto_round/autoround.py@@ -191,6 +191,8 @@ class AutoRound(object):enable_quanted_input: bool = kwargs.pop("enable_quanted_input", True)disable_deterministic_algorithms = kwargs.pop("disable_deterministic_algorithms", False)if not disable_deterministic_algorithms:+ if "CUBLAS_WORKSPACE_CONFIG" not in os.environ:+ os.environ["CUBLAS_WORKSPACE_CONFIG"] = ":4096:8"torch.use_deterministic_algorithms(True, warn_only=False)self.vlm = kwargs.pop("vlm") if "vlm" in kwargs else Falsediff --git a/auto_round/script/llm.py b/auto_round/script/llm.pyindex 520c13610..9b83aa371 100644--- a/auto_round/script/llm.py+++ b/auto_round/script/llm.py@@ -42,8 +42,6 @@ from auto_round.utils import (str2bool,)-os.environ["CUBLAS_WORKSPACE_CONFIG"] = ":4096:8"-class BasicArgumentParser(argparse.ArgumentParser):@@ -430,11 +428,6 @@ def tune(args):import torch- if not args.disable_deterministic_algorithms:- torch.use_deterministic_algorithms(True, warn_only=False)- # logger.info("`torch.use_deterministic_algorithms` is enabled by default for reproducibility "- # "and can be disabled using the `--disable_deterministic_algorithms` argument.")-if args.enable_torch_compile:logger.info("`torch.compile` is enabled to reduce tuning costs. "@@ -571,6 +564,7 @@ def tune(args):super_group_size=args.super_group_size,super_bits=args.super_bits,disable_opt_rtn=args.disable_opt_rtn,+ disable_deterministic_algorithms=args.disable_deterministic_algorithms,**mllm_kwargs,)diff --git a/auto_round/script/mllm.py b/auto_round/script/mllm.pyindex 3864d4abc..86c850628 100644--- a/auto_round/script/mllm.py+++ b/auto_round/script/mllm.py@@ -25,8 +25,6 @@ from auto_round.utils import (set_cuda_visible_devices,)-os.environ["CUBLAS_WORKSPACE_CONFIG"] = ":4096:8"-class BasicArgumentParser(argparse.ArgumentParser):@@ -331,13 +329,6 @@ def tune(args):import torch- if not args.disable_deterministic_algorithms:- torch.use_deterministic_algorithms(True, warn_only=False)- print(- "'torch.use_deterministic_algorithms' is turned on by default for reproducibility, "- "and can be turned off by setting the '--disable_deterministic_algorithms' parameter."- )-model_name = args.modelif model_name[-1] == "/":model_name = model_name[:-1]@@ -484,6 +475,7 @@ def tune(args):model_kwargs=model_kwargs,data_type=args.data_type,disable_opt_rtn=args.disable_opt_rtn,+ disable_deterministic_algorithms=args.disable_deterministic_algorithms,)model_name = args.model.rstrip("/")diff --git a/auto_round_extension/torch/qlinear_torch_zp.py b/auto_round_extension/torch/qlinear_torch_zp.pyindex 32747de0d..2958d249c 100644--- a/auto_round_extension/torch/qlinear_torch_zp.py+++ b/auto_round_extension/torch/qlinear_torch_zp.py@@ -244,7 +244,7 @@ class QuantLinear(nn.Module):weight[:, 1, 11] = (weight[:, 1, 11] & 0x1) | ((weight[:, 2, 0] << 1) & 0x6)weight = weight & 0x7weight = torch.cat([weight[:, 0, :11], weight[:, 1, 1:12], weight[:, 2, 1:11]], dim=1)-+ zeros += 1weight = weight.reshape(weight.shape[0] * weight.shape[1], weight.shape[2])if hasattr(self, "g_idx"):num_itr = self.g_idx.shape[0] // x.shape[-1]diff --git a/test/test_cuda/test_torch_backend.py b/test/test_cuda/test_torch_backend.pynew file mode 100644index 000000000..562527f92--- /dev/null+++ b/test/test_cuda/test_torch_backend.py@@ -0,0 +1,141 @@+import shutil+import sys+import unittest++import pytest++sys.path.insert(0, "../..")+++import torch+from transformers import AutoModelForCausalLM, AutoTokenizer++from auto_round import AutoRound, AutoRoundConfig+from auto_round.eval.evaluation import simple_evaluate_user_model+from auto_round.testing_utils import require_autogptq, require_gptqmodel+++class LLMDataLoader:+ def __init__(self):+ self.batch_size = 1++ def __iter__(self):+ for i in range(2):+ yield torch.ones([1, 10], dtype=torch.long)+++class TestAutoRoundTorchBackend(unittest.TestCase):++ @classmethod+ def setUpClass(self):+ self.model_name = "/models/opt-125m"+ self.save_folder = "./saved"+ self.llm_dataloader = LLMDataLoader()++ def model_infer(self, model, tokenizer):+ prompts = [+ "Hello,my name is",+ # "The president of the United States is",+ # "The capital of France is",+ # "The future of AI is",+ ]++ inputs = tokenizer(prompts, return_tensors="pt", padding=False, truncation=True)++ outputs = model.generate(+ input_ids=inputs["input_ids"].to(model.device),+ attention_mask=inputs["attention_mask"].to(model.device),+ do_sample=False, ## change this to follow official usage+ max_new_tokens=5,+ )+ generated_ids = [output_ids[len(input_ids) :] for input_ids, output_ids in zip(inputs["input_ids"], outputs)]++ decoded_outputs = tokenizer.batch_decode(generated_ids, skip_special_tokens=True)++ for i, prompt in enumerate(prompts):+ print(f"Prompt: {prompt}")+ print(f"Generated: {decoded_outputs[i]}")+ print("-" * 50)+ return decoded_outputs[0]++ @classmethod+ def tearDownClass(self):+ shutil.rmtree(self.save_folder, ignore_errors=True)+ shutil.rmtree("runs", ignore_errors=True)++ def test_torch_4bits_asym(self):+ model = AutoModelForCausalLM.from_pretrained(self.model_name, torch_dtype="auto", trust_remote_code=True)+ tokenizer = AutoTokenizer.from_pretrained(self.model_name, trust_remote_code=True)+ bits, group_size, sym = 4, 128, False+ autoround = AutoRound(+ model,+ tokenizer,+ bits=bits,+ group_size=group_size,+ sym=sym,+ iters=1,+ seqlen=2,+ dataset=self.llm_dataloader,+ disable_deterministic_algorithms=True,+ )+ quantized_model_path = self.save_folder+ autoround.quantize_and_save(output_dir=quantized_model_path, format="auto_round:gptqmodel")++ quantization_config = AutoRoundConfig(backend="torch")+ model = AutoModelForCausalLM.from_pretrained(+ quantized_model_path, torch_dtype=torch.float16, device_map="auto", quantization_config=quantization_config+ )++ tokenizer = AutoTokenizer.from_pretrained(self.save_folder)+ self.model_infer(model, tokenizer)+ result = simple_evaluate_user_model(model, tokenizer, batch_size=16, tasks="lambada_openai")+ print(result["results"]["lambada_openai"]["acc,none"])+ self.assertGreater(result["results"]["lambada_openai"]["acc,none"], 0.35)+ torch.cuda.empty_cache()++ model = AutoModelForCausalLM.from_pretrained(+ self.save_folder, torch_dtype=torch.bfloat16, device_map="auto", quantization_config=quantization_config+ )++ tokenizer = AutoTokenizer.from_pretrained(self.save_folder)+ self.model_infer(model, tokenizer)+ result = simple_evaluate_user_model(model, tokenizer, batch_size=16, tasks="lambada_openai")+ print(result["results"]["lambada_openai"]["acc,none"])+ self.assertGreater(result["results"]["lambada_openai"]["acc,none"], 0.35)+ torch.cuda.empty_cache()+ shutil.rmtree("./saved", ignore_errors=True)++ def test_torch_4bits_sym(self):+ model = AutoModelForCausalLM.from_pretrained(self.model_name, torch_dtype="auto", trust_remote_code=True)+ tokenizer = AutoTokenizer.from_pretrained(self.model_name, trust_remote_code=True)+ bits, group_size, sym = 4, 128, True+ autoround = AutoRound(+ model,+ tokenizer,+ bits=bits,+ group_size=group_size,+ sym=sym,+ iters=1,+ seqlen=2,+ dataset=self.llm_dataloader,+ # disable_deterministic_algorithms=True+ )+ quantized_model_path = self.save_folder+ autoround.quantize_and_save(output_dir=quantized_model_path, format="auto_round") ##will convert to gptq model++ quantization_config = AutoRoundConfig(backend="torch")+ model = AutoModelForCausalLM.from_pretrained(+ quantized_model_path, torch_dtype=torch.float16, device_map="auto", quantization_config=quantization_config+ )++ tokenizer = AutoTokenizer.from_pretrained(self.save_folder)+ self.model_infer(model, tokenizer)+ result = simple_evaluate_user_model(model, tokenizer, batch_size=16, tasks="lambada_openai")+ print(result["results"]["lambada_openai"]["acc,none"])+ self.assertGreater(result["results"]["lambada_openai"]["acc,none"], 0.29)+ torch.cuda.empty_cache()+ shutil.rmtree(self.save_folder, ignore_errors=True)+++if __name__ == "__main__":+ unittest.main()