Skip to content

PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度

Fix torch_zp infer bug & API disable_deterministic_algorithms bug (#733)

Weiwei71dab1dca142
diff --git a/auto_round/autoround.py b/auto_round/autoround.py
index b143cac36..3c0b07247 100644
--- a/auto_round/autoround.py
+++ b/auto_round/autoround.py
@@ -191,6 +191,8 @@ class AutoRound(object):
enable_quanted_input: bool = kwargs.pop("enable_quanted_input", True)
disable_deterministic_algorithms = kwargs.pop("disable_deterministic_algorithms", False)
if not disable_deterministic_algorithms:
+ if "CUBLAS_WORKSPACE_CONFIG" not in os.environ:
+ os.environ["CUBLAS_WORKSPACE_CONFIG"] = ":4096:8"
torch.use_deterministic_algorithms(True, warn_only=False)
self.vlm = kwargs.pop("vlm") if "vlm" in kwargs else False
diff --git a/auto_round/script/llm.py b/auto_round/script/llm.py
index 520c13610..9b83aa371 100644
--- a/auto_round/script/llm.py
+++ b/auto_round/script/llm.py
@@ -42,8 +42,6 @@ from auto_round.utils import (
str2bool,
)
-os.environ["CUBLAS_WORKSPACE_CONFIG"] = ":4096:8"
-
class BasicArgumentParser(argparse.ArgumentParser):
@@ -430,11 +428,6 @@ def tune(args):
import torch
- if not args.disable_deterministic_algorithms:
- torch.use_deterministic_algorithms(True, warn_only=False)
- # logger.info("`torch.use_deterministic_algorithms` is enabled by default for reproducibility "
- # "and can be disabled using the `--disable_deterministic_algorithms` argument.")
-
if args.enable_torch_compile:
logger.info(
"`torch.compile` is enabled to reduce tuning costs. "
@@ -571,6 +564,7 @@ def tune(args):
super_group_size=args.super_group_size,
super_bits=args.super_bits,
disable_opt_rtn=args.disable_opt_rtn,
+ disable_deterministic_algorithms=args.disable_deterministic_algorithms,
**mllm_kwargs,
)
diff --git a/auto_round/script/mllm.py b/auto_round/script/mllm.py
index 3864d4abc..86c850628 100644
--- a/auto_round/script/mllm.py
+++ b/auto_round/script/mllm.py
@@ -25,8 +25,6 @@ from auto_round.utils import (
set_cuda_visible_devices,
)
-os.environ["CUBLAS_WORKSPACE_CONFIG"] = ":4096:8"
-
class BasicArgumentParser(argparse.ArgumentParser):
@@ -331,13 +329,6 @@ def tune(args):
import torch
- if not args.disable_deterministic_algorithms:
- torch.use_deterministic_algorithms(True, warn_only=False)
- print(
- "'torch.use_deterministic_algorithms' is turned on by default for reproducibility, "
- "and can be turned off by setting the '--disable_deterministic_algorithms' parameter."
- )
-
model_name = args.model
if model_name[-1] == "/":
model_name = model_name[:-1]
@@ -484,6 +475,7 @@ def tune(args):
model_kwargs=model_kwargs,
data_type=args.data_type,
disable_opt_rtn=args.disable_opt_rtn,
+ disable_deterministic_algorithms=args.disable_deterministic_algorithms,
)
model_name = args.model.rstrip("/")
diff --git a/auto_round_extension/torch/qlinear_torch_zp.py b/auto_round_extension/torch/qlinear_torch_zp.py
index 32747de0d..2958d249c 100644
--- a/auto_round_extension/torch/qlinear_torch_zp.py
+++ b/auto_round_extension/torch/qlinear_torch_zp.py
@@ -244,7 +244,7 @@ class QuantLinear(nn.Module):
weight[:, 1, 11] = (weight[:, 1, 11] & 0x1) | ((weight[:, 2, 0] << 1) & 0x6)
weight = weight & 0x7
weight = torch.cat([weight[:, 0, :11], weight[:, 1, 1:12], weight[:, 2, 1:11]], dim=1)
-
+ zeros += 1
weight = weight.reshape(weight.shape[0] * weight.shape[1], weight.shape[2])
if hasattr(self, "g_idx"):
num_itr = self.g_idx.shape[0] // x.shape[-1]
diff --git a/test/test_cuda/test_torch_backend.py b/test/test_cuda/test_torch_backend.py
new file mode 100644
index 000000000..562527f92
--- /dev/null
+++ b/test/test_cuda/test_torch_backend.py
@@ -0,0 +1,141 @@
+import shutil
+import sys
+import unittest
+
+import pytest
+
+sys.path.insert(0, "../..")
+
+
+import torch
+from transformers import AutoModelForCausalLM, AutoTokenizer
+
+from auto_round import AutoRound, AutoRoundConfig
+from auto_round.eval.evaluation import simple_evaluate_user_model
+from auto_round.testing_utils import require_autogptq, require_gptqmodel
+
+
+class LLMDataLoader:
+ def __init__(self):
+ self.batch_size = 1
+
+ def __iter__(self):
+ for i in range(2):
+ yield torch.ones([1, 10], dtype=torch.long)
+
+
+class TestAutoRoundTorchBackend(unittest.TestCase):
+
+ @classmethod
+ def setUpClass(self):
+ self.model_name = "/models/opt-125m"
+ self.save_folder = "./saved"
+ self.llm_dataloader = LLMDataLoader()
+
+ def model_infer(self, model, tokenizer):
+ prompts = [
+ "Hello,my name is",
+ # "The president of the United States is",
+ # "The capital of France is",
+ # "The future of AI is",
+ ]
+
+ inputs = tokenizer(prompts, return_tensors="pt", padding=False, truncation=True)
+
+ outputs = model.generate(
+ input_ids=inputs["input_ids"].to(model.device),
+ attention_mask=inputs["attention_mask"].to(model.device),
+ do_sample=False, ## change this to follow official usage
+ max_new_tokens=5,
+ )
+ generated_ids = [output_ids[len(input_ids) :] for input_ids, output_ids in zip(inputs["input_ids"], outputs)]
+
+ decoded_outputs = tokenizer.batch_decode(generated_ids, skip_special_tokens=True)
+
+ for i, prompt in enumerate(prompts):
+ print(f"Prompt: {prompt}")
+ print(f"Generated: {decoded_outputs[i]}")
+ print("-" * 50)
+ return decoded_outputs[0]
+
+ @classmethod
+ def tearDownClass(self):
+ shutil.rmtree(self.save_folder, ignore_errors=True)
+ shutil.rmtree("runs", ignore_errors=True)
+
+ def test_torch_4bits_asym(self):
+ model = AutoModelForCausalLM.from_pretrained(self.model_name, torch_dtype="auto", trust_remote_code=True)
+ tokenizer = AutoTokenizer.from_pretrained(self.model_name, trust_remote_code=True)
+ bits, group_size, sym = 4, 128, False
+ autoround = AutoRound(
+ model,
+ tokenizer,
+ bits=bits,
+ group_size=group_size,
+ sym=sym,
+ iters=1,
+ seqlen=2,
+ dataset=self.llm_dataloader,
+ disable_deterministic_algorithms=True,
+ )
+ quantized_model_path = self.save_folder
+ autoround.quantize_and_save(output_dir=quantized_model_path, format="auto_round:gptqmodel")
+
+ quantization_config = AutoRoundConfig(backend="torch")
+ model = AutoModelForCausalLM.from_pretrained(
+ quantized_model_path, torch_dtype=torch.float16, device_map="auto", quantization_config=quantization_config
+ )
+
+ tokenizer = AutoTokenizer.from_pretrained(self.save_folder)
+ self.model_infer(model, tokenizer)
+ result = simple_evaluate_user_model(model, tokenizer, batch_size=16, tasks="lambada_openai")
+ print(result["results"]["lambada_openai"]["acc,none"])
+ self.assertGreater(result["results"]["lambada_openai"]["acc,none"], 0.35)
+ torch.cuda.empty_cache()
+
+ model = AutoModelForCausalLM.from_pretrained(
+ self.save_folder, torch_dtype=torch.bfloat16, device_map="auto", quantization_config=quantization_config
+ )
+
+ tokenizer = AutoTokenizer.from_pretrained(self.save_folder)
+ self.model_infer(model, tokenizer)
+ result = simple_evaluate_user_model(model, tokenizer, batch_size=16, tasks="lambada_openai")
+ print(result["results"]["lambada_openai"]["acc,none"])
+ self.assertGreater(result["results"]["lambada_openai"]["acc,none"], 0.35)
+ torch.cuda.empty_cache()
+ shutil.rmtree("./saved", ignore_errors=True)
+
+ def test_torch_4bits_sym(self):
+ model = AutoModelForCausalLM.from_pretrained(self.model_name, torch_dtype="auto", trust_remote_code=True)
+ tokenizer = AutoTokenizer.from_pretrained(self.model_name, trust_remote_code=True)
+ bits, group_size, sym = 4, 128, True
+ autoround = AutoRound(
+ model,
+ tokenizer,
+ bits=bits,
+ group_size=group_size,
+ sym=sym,
+ iters=1,
+ seqlen=2,
+ dataset=self.llm_dataloader,
+ # disable_deterministic_algorithms=True
+ )
+ quantized_model_path = self.save_folder
+ autoround.quantize_and_save(output_dir=quantized_model_path, format="auto_round") ##will convert to gptq model
+
+ quantization_config = AutoRoundConfig(backend="torch")
+ model = AutoModelForCausalLM.from_pretrained(
+ quantized_model_path, torch_dtype=torch.float16, device_map="auto", quantization_config=quantization_config
+ )
+
+ tokenizer = AutoTokenizer.from_pretrained(self.save_folder)
+ self.model_infer(model, tokenizer)
+ result = simple_evaluate_user_model(model, tokenizer, batch_size=16, tasks="lambada_openai")
+ print(result["results"]["lambada_openai"]["acc,none"])
+ self.assertGreater(result["results"]["lambada_openai"]["acc,none"], 0.29)
+ torch.cuda.empty_cache()
+ shutil.rmtree(self.save_folder, ignore_errors=True)
+
+
+if __name__ == "__main__":
+ unittest.main()