PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度
fix mllm dataset config bug and add gptq cuda backend (#329)
Wenhua Cheng56e34003ece3
diff --git a/auto_round/auto_quantizer.py b/auto_round/auto_quantizer.pyindex b58c84e7b..559c30ad4 100644--- a/auto_round/auto_quantizer.py+++ b/auto_round/auto_quantizer.py@@ -469,21 +469,10 @@ class AutoRoundQuantizer(HfQuantizer):If any condition related to backend or quantization configuration is not met."""- def remove_str(input_string: str, sub_str) -> str:- """Removes the specified substring from the input string, if present.-- Args:- input_string (str):- The original string from which to remove the substring.- sub_str (str):- The substring to be removed.-- Returns:- str:- The modified string with the substring removed.- """- pattern = re.escape(sub_str) + r':?'- return re.sub(pattern, '', input_string)+ def remove_device_str(s, device_str):+ if s and s.startswith(device_str):+ return s[len(device_str):].lstrip(":")+ return sif "auto" == target_backend.split(':')[0]:target_backend = target_backend[4:] # Remove 'auto'@@ -491,9 +480,9 @@ class AutoRoundQuantizer(HfQuantizer):target_backend = target_backend[1:]# Remove device info from target_backend- target_backend = remove_str(target_backend, "cpu")- target_backend = remove_str(target_backend, "hpu")- target_backend = remove_str(target_backend, "cuda")+ target_backend = remove_device_str(target_backend, "cpu")+ target_backend = remove_device_str(target_backend, "hpu")+ target_backend = remove_device_str(target_backend, "cuda")orig_backend = self.find_backend(orig_backend)if target_backend == "":@@ -745,4 +734,3 @@ if version.parse(transformers.__version__) < version.parse("4.38.0"):transformers.quantizers.auto.AutoHfQuantizer = AutoHfQuantizertransformers.modeling_utils.AutoHfQuantizer = AutoHfQuantizer-diff --git a/auto_round/autoround.py b/auto_round/autoround.pyindex fe904016b..9fe822129 100644--- a/auto_round/autoround.py+++ b/auto_round/autoround.py@@ -257,11 +257,13 @@ class AutoRound(object):if "mx_fp" in self.data_type and self.group_size != 32:logger.warning("mx_fp should only support group_size of 32 in real deployment")- if self.batch_size > self.nsamples:++ if self.nsamples < self.gradient_accumulate_steps * self.batch_size:+ self.batch_size = min(self.batch_size, self.nsamples)+ self.gradient_accumulate_steps = min(self.nsamples // self.batch_size, self.gradient_accumulate_steps)logger.warning(- f"reset batch_size to nsamples({self.nsamples}) "- f"as batch_size({self.batch_size}) must be smaller than nsamples")- self.batch_size = self.nsamples+ f"reset gradient_accumulate_steps to {self.gradient_accumulate_steps} as nsamples must equal or greater"+ " than gradient_accumulate_steps * batch_szie")def quantize(self):"""Quantize the model and return the quantized model along with layer configurations.diff --git a/auto_round/backend.py b/auto_round/backend.pyindex 66c5e6671..da5690c0c 100644--- a/auto_round/backend.py+++ b/auto_round/backend.py@@ -119,6 +119,15 @@ BackendInfos['gptq:tritonv2'] = BackendInfo(device=["cuda"], sym=[True, False],requirements=["auto-gptq>=0.7.1"])+BackendInfos['gptq:cuda'] = BackendInfo(device=["cuda"], sym=[True, False],+ packing_format="triton_zp+-1",+ bits=[2, 3, 4, 8], group_size=None,+ priority=0, feature_checks=[feature_multiply_checker_32],+ alias=["auto_round:auto_gptq:cuda,auto_gptq:cuda, auto_round:gptq:cuda"],+ convertable_format=["triton_zp+-1"],+ requirements=["auto-gptq>=0.7.1"]+ )+BackendInfos['awq:gemm'] = BackendInfo(device=["cuda"], sym=[True, False], ##actrally is gemmpacking_format="awq",bits=[4], group_size=None,diff --git a/auto_round/calib_dataset.py b/auto_round/calib_dataset.pyindex 82dfa66c4..aad25b840 100644--- a/auto_round/calib_dataset.py+++ b/auto_round/calib_dataset.py@@ -16,7 +16,6 @@ import jsonimport randomimport torch-torch.use_deterministic_algorithms(True, warn_only=True)from torch.utils.data import DataLoaderdiff --git a/auto_round/mllm/autoround_mllm.py b/auto_round/mllm/autoround_mllm.pyindex 485aa0d15..1f23cf4a7 100644--- a/auto_round/mllm/autoround_mllm.py+++ b/auto_round/mllm/autoround_mllm.py@@ -28,7 +28,7 @@ from ..autoround import AutoRoundfrom .template import get_template, Templatefrom .mllm_dataset import get_mllm_dataloaderfrom ..low_cpu_mem.utils import get_layers_before_block-from ..calib_dataset import CALIB_DATASETS+def _only_text_test(model, tokenizer):@@ -36,6 +36,8 @@ def _only_text_test(model, tokenizer):try:text = ["only text", "test"]tokenizer.padding_side = 'left'+ if tokenizer.pad_token is None:+ tokenizer.pad_token = tokenizer.eos_tokeninputs = tokenizer(text, return_tensors="pt", padding=True, truncation=True).to(model.device)model(**inputs)return True@@ -145,12 +147,21 @@ class AutoRoundMLLM(AutoRound):self.template, model=model, tokenizer=tokenizer, image_processor=image_processor)dataset = self.template.default_dataset if dataset is None else dataset+ from ..calib_dataset import CALIB_DATASETSif truncation is None:truncation = True if dataset in CALIB_DATASETS.keys() else Falseself.truncation = truncation- if dataset in CALIB_DATASETS.keys() and not _only_text_test(model, tokenizer):- logger.warning(f"{model.config.model_type} not support for {dataset},"+ if nsamples % batch_size != 0:+ nsamples = (nsamples // batch_size + 1) * batch_size+ logger.warning(f"'nsamples' is not divisible by 'batch_size', will adjusted to {nsamples}")++ if quant_nontext_module or (dataset in CALIB_DATASETS.keys() and not _only_text_test(model, tokenizer)):+ if quant_nontext_module:+ logger.warning(f"Quantitative nontext module is not supported for plain text datasets,"+ "will use liuhaotian/llava_conv_58k with default config as an alternative.")+ else:+ logger.warning(f"{model.config.model_type} not support for {dataset},"" will use liuhaotian/llava_conv_58k with default config as an alternative.")dataset = "liuhaotian/llava_conv_58k"self.truncation = False@@ -318,6 +329,15 @@ class AutoRoundMLLM(AutoRound):f"Insufficient number of samples collected may affect the quantification. "f"target samples count is {nsamples}, while valid samples count is {total_cnt}")+ if total_cnt < self.batch_size:+ raise ValueError(f"valid samples is less than batch_size({self.batch_size}),"+ " please adjust self.batch_size or seqlen.")+ max_len = (total_cnt // self.batch_size) * self.batch_size+ for k, v in self.inputs.items():+ for key in v:+ if isinstance(v[key], list) and len(v[key]) == total_cnt:+ self.inputs[k][key] = v[key][:max_len]+# clean embed weight to save memoryif self.low_cpu_mem_usage:diff --git a/test/test_autoround.py b/test/test_autoround.pyindex da857531d..5e831aca4 100644--- a/test/test_autoround.py+++ b/test/test_autoround.py@@ -16,7 +16,7 @@ class LLMDataLoader:self.batch_size = 1def __iter__(self):- for i in range(2):+ for i in range(3):yield torch.ones([1, 10], dtype=torch.long)@@ -100,6 +100,19 @@ class TestAutoRound(unittest.TestCase):)autoround.quantize()+ def test_nsample(self):+ autoround= AutoRound(+ self.model,+ self.tokenizer,+ bits=4,+ group_size=128,+ seqlen=2,+ nsamples=3,+ batch_size=3,+ iters=2,+ dataset=self.llm_dataloader,+ gradient_accumulate_steps=4)+ autoround.quantize()def test_default(self):bits, group_size, sym = 4, 128, Falsediff --git a/test/test_basic_usage.py b/test/test_basic_usage.pyindex 9664b2eb3..6d28801db 100644--- a/test/test_basic_usage.py+++ b/test/test_basic_usage.py@@ -19,7 +19,7 @@ class TestAutoRoundCmd(unittest.TestCase):def test_auto_round_cmd(self):python_path = sys.executable- # test llm script+ ##test llm scriptres = os.system(f"cd .. && {python_path} -m auto_round -h")if res > 0 or res == -1:@@ -29,8 +29,8 @@ class TestAutoRoundCmd(unittest.TestCase):f"cd .. && {python_path} -m auto_round --model 'facebook/opt-125m' --iter 2 --nsamples 1 --format auto_gptq,auto_round --disable_eval --output_dir ./saved")if res > 0 or res == -1:assert False, "cmd line test fail, please have a check"--++# test mllm scriptres = os.system(f"cd .. && {python_path} -m auto_round --mllm -h")@@ -41,10 +41,10 @@ class TestAutoRoundCmd(unittest.TestCase):f"cd .. && {python_path} -m auto_round --mllm --iter 2 --nsamples 10 --format auto_round --output_dir ./saved")if res > 0 or res == -1:assert False, "cmd line test fail, please have a check"-+res = os.system(f"cd .. && {python_path} -m auto_round --mllm --iter 2 --nsamples 10 --format auto_round"- " --quant_nontext_module --output_dir ./saved")+ " --quant_nontext_module --output_dir ./saved ")if res > 0 or res == -1:assert False, "cmd line test fail, please have a check"diff --git a/test/test_mllm.py b/test/test_mllm.pyindex ef3b62396..3441962d3 100644--- a/test/test_mllm.py+++ b/test/test_mllm.py@@ -22,7 +22,7 @@ class FakeDataLoader:}def __iter__(self):- for i in range(2):+ for i in range(4):yield self.data@@ -63,8 +63,8 @@ class TestAutoRoundMLLM(unittest.TestCase):bits, group_size = 4, 128autoround = AutoRoundMLLM(model, tokenizer, bits=bits, group_size=group_size,- nsamples=1,- batch_size=1, iters=2, dataset=self.dataset, quant_nontext_module=True,seqlen=256)+ nsamples=5,+ batch_size=3, iters=2, dataset=self.dataset, quant_nontext_module=False,seqlen=256)autoround.quantize()autoround.save_quantized("./saved/", format="auto_round", inplace=True)