Skip to content

PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度

fix mllm dataset config bug and add gptq cuda backend (#329)

Wenhua Cheng56e34003ece3
diff --git a/auto_round/auto_quantizer.py b/auto_round/auto_quantizer.py
index b58c84e7b..559c30ad4 100644
--- a/auto_round/auto_quantizer.py
+++ b/auto_round/auto_quantizer.py
@@ -469,21 +469,10 @@ class AutoRoundQuantizer(HfQuantizer):
If any condition related to backend or quantization configuration is not met.
"""
- def remove_str(input_string: str, sub_str) -> str:
- """Removes the specified substring from the input string, if present.
-
- Args:
- input_string (str):
- The original string from which to remove the substring.
- sub_str (str):
- The substring to be removed.
-
- Returns:
- str:
- The modified string with the substring removed.
- """
- pattern = re.escape(sub_str) + r':?'
- return re.sub(pattern, '', input_string)
+ def remove_device_str(s, device_str):
+ if s and s.startswith(device_str):
+ return s[len(device_str):].lstrip(":")
+ return s
if "auto" == target_backend.split(':')[0]:
target_backend = target_backend[4:] # Remove 'auto'
@@ -491,9 +480,9 @@ class AutoRoundQuantizer(HfQuantizer):
target_backend = target_backend[1:]
# Remove device info from target_backend
- target_backend = remove_str(target_backend, "cpu")
- target_backend = remove_str(target_backend, "hpu")
- target_backend = remove_str(target_backend, "cuda")
+ target_backend = remove_device_str(target_backend, "cpu")
+ target_backend = remove_device_str(target_backend, "hpu")
+ target_backend = remove_device_str(target_backend, "cuda")
orig_backend = self.find_backend(orig_backend)
if target_backend == "":
@@ -745,4 +734,3 @@ if version.parse(transformers.__version__) < version.parse("4.38.0"):
transformers.quantizers.auto.AutoHfQuantizer = AutoHfQuantizer
transformers.modeling_utils.AutoHfQuantizer = AutoHfQuantizer
-
diff --git a/auto_round/autoround.py b/auto_round/autoround.py
index fe904016b..9fe822129 100644
--- a/auto_round/autoround.py
+++ b/auto_round/autoround.py
@@ -257,11 +257,13 @@ class AutoRound(object):
if "mx_fp" in self.data_type and self.group_size != 32:
logger.warning("mx_fp should only support group_size of 32 in real deployment")
- if self.batch_size > self.nsamples:
+
+ if self.nsamples < self.gradient_accumulate_steps * self.batch_size:
+ self.batch_size = min(self.batch_size, self.nsamples)
+ self.gradient_accumulate_steps = min(self.nsamples // self.batch_size, self.gradient_accumulate_steps)
logger.warning(
- f"reset batch_size to nsamples({self.nsamples}) "
- f"as batch_size({self.batch_size}) must be smaller than nsamples")
- self.batch_size = self.nsamples
+ f"reset gradient_accumulate_steps to {self.gradient_accumulate_steps} as nsamples must equal or greater"
+ " than gradient_accumulate_steps * batch_szie")
def quantize(self):
"""Quantize the model and return the quantized model along with layer configurations.
diff --git a/auto_round/backend.py b/auto_round/backend.py
index 66c5e6671..da5690c0c 100644
--- a/auto_round/backend.py
+++ b/auto_round/backend.py
@@ -119,6 +119,15 @@ BackendInfos['gptq:tritonv2'] = BackendInfo(device=["cuda"], sym=[True, False],
requirements=["auto-gptq>=0.7.1"]
)
+BackendInfos['gptq:cuda'] = BackendInfo(device=["cuda"], sym=[True, False],
+ packing_format="triton_zp+-1",
+ bits=[2, 3, 4, 8], group_size=None,
+ priority=0, feature_checks=[feature_multiply_checker_32],
+ alias=["auto_round:auto_gptq:cuda,auto_gptq:cuda, auto_round:gptq:cuda"],
+ convertable_format=["triton_zp+-1"],
+ requirements=["auto-gptq>=0.7.1"]
+ )
+
BackendInfos['awq:gemm'] = BackendInfo(device=["cuda"], sym=[True, False], ##actrally is gemm
packing_format="awq",
bits=[4], group_size=None,
diff --git a/auto_round/calib_dataset.py b/auto_round/calib_dataset.py
index 82dfa66c4..aad25b840 100644
--- a/auto_round/calib_dataset.py
+++ b/auto_round/calib_dataset.py
@@ -16,7 +16,6 @@ import json
import random
import torch
-
torch.use_deterministic_algorithms(True, warn_only=True)
from torch.utils.data import DataLoader
diff --git a/auto_round/mllm/autoround_mllm.py b/auto_round/mllm/autoround_mllm.py
index 485aa0d15..1f23cf4a7 100644
--- a/auto_round/mllm/autoround_mllm.py
+++ b/auto_round/mllm/autoround_mllm.py
@@ -28,7 +28,7 @@ from ..autoround import AutoRound
from .template import get_template, Template
from .mllm_dataset import get_mllm_dataloader
from ..low_cpu_mem.utils import get_layers_before_block
-from ..calib_dataset import CALIB_DATASETS
+
def _only_text_test(model, tokenizer):
@@ -36,6 +36,8 @@ def _only_text_test(model, tokenizer):
try:
text = ["only text", "test"]
tokenizer.padding_side = 'left'
+ if tokenizer.pad_token is None:
+ tokenizer.pad_token = tokenizer.eos_token
inputs = tokenizer(text, return_tensors="pt", padding=True, truncation=True).to(model.device)
model(**inputs)
return True
@@ -145,12 +147,21 @@ class AutoRoundMLLM(AutoRound):
self.template, model=model, tokenizer=tokenizer, image_processor=image_processor)
dataset = self.template.default_dataset if dataset is None else dataset
+ from ..calib_dataset import CALIB_DATASETS
if truncation is None:
truncation = True if dataset in CALIB_DATASETS.keys() else False
self.truncation = truncation
- if dataset in CALIB_DATASETS.keys() and not _only_text_test(model, tokenizer):
- logger.warning(f"{model.config.model_type} not support for {dataset},"
+ if nsamples % batch_size != 0:
+ nsamples = (nsamples // batch_size + 1) * batch_size
+ logger.warning(f"'nsamples' is not divisible by 'batch_size', will adjusted to {nsamples}")
+
+ if quant_nontext_module or (dataset in CALIB_DATASETS.keys() and not _only_text_test(model, tokenizer)):
+ if quant_nontext_module:
+ logger.warning(f"Quantitative nontext module is not supported for plain text datasets,"
+ "will use liuhaotian/llava_conv_58k with default config as an alternative.")
+ else:
+ logger.warning(f"{model.config.model_type} not support for {dataset},"
" will use liuhaotian/llava_conv_58k with default config as an alternative.")
dataset = "liuhaotian/llava_conv_58k"
self.truncation = False
@@ -318,6 +329,15 @@ class AutoRoundMLLM(AutoRound):
f"Insufficient number of samples collected may affect the quantification. "
f"target samples count is {nsamples}, while valid samples count is {total_cnt}"
)
+ if total_cnt < self.batch_size:
+ raise ValueError(f"valid samples is less than batch_size({self.batch_size}),"
+ " please adjust self.batch_size or seqlen.")
+ max_len = (total_cnt // self.batch_size) * self.batch_size
+ for k, v in self.inputs.items():
+ for key in v:
+ if isinstance(v[key], list) and len(v[key]) == total_cnt:
+ self.inputs[k][key] = v[key][:max_len]
+
# clean embed weight to save memory
if self.low_cpu_mem_usage:
diff --git a/test/test_autoround.py b/test/test_autoround.py
index da857531d..5e831aca4 100644
--- a/test/test_autoround.py
+++ b/test/test_autoround.py
@@ -16,7 +16,7 @@ class LLMDataLoader:
self.batch_size = 1
def __iter__(self):
- for i in range(2):
+ for i in range(3):
yield torch.ones([1, 10], dtype=torch.long)
@@ -100,6 +100,19 @@ class TestAutoRound(unittest.TestCase):
)
autoround.quantize()
+ def test_nsample(self):
+ autoround= AutoRound(
+ self.model,
+ self.tokenizer,
+ bits=4,
+ group_size=128,
+ seqlen=2,
+ nsamples=3,
+ batch_size=3,
+ iters=2,
+ dataset=self.llm_dataloader,
+ gradient_accumulate_steps=4)
+ autoround.quantize()
def test_default(self):
bits, group_size, sym = 4, 128, False
diff --git a/test/test_basic_usage.py b/test/test_basic_usage.py
index 9664b2eb3..6d28801db 100644
--- a/test/test_basic_usage.py
+++ b/test/test_basic_usage.py
@@ -19,7 +19,7 @@ class TestAutoRoundCmd(unittest.TestCase):
def test_auto_round_cmd(self):
python_path = sys.executable
- # test llm script
+ ##test llm script
res = os.system(
f"cd .. && {python_path} -m auto_round -h")
if res > 0 or res == -1:
@@ -29,8 +29,8 @@ class TestAutoRoundCmd(unittest.TestCase):
f"cd .. && {python_path} -m auto_round --model 'facebook/opt-125m' --iter 2 --nsamples 1 --format auto_gptq,auto_round --disable_eval --output_dir ./saved")
if res > 0 or res == -1:
assert False, "cmd line test fail, please have a check"
-
-
+
+
# test mllm script
res = os.system(
f"cd .. && {python_path} -m auto_round --mllm -h")
@@ -41,10 +41,10 @@ class TestAutoRoundCmd(unittest.TestCase):
f"cd .. && {python_path} -m auto_round --mllm --iter 2 --nsamples 10 --format auto_round --output_dir ./saved")
if res > 0 or res == -1:
assert False, "cmd line test fail, please have a check"
-
+
res = os.system(
f"cd .. && {python_path} -m auto_round --mllm --iter 2 --nsamples 10 --format auto_round"
- " --quant_nontext_module --output_dir ./saved")
+ " --quant_nontext_module --output_dir ./saved ")
if res > 0 or res == -1:
assert False, "cmd line test fail, please have a check"
diff --git a/test/test_mllm.py b/test/test_mllm.py
index ef3b62396..3441962d3 100644
--- a/test/test_mllm.py
+++ b/test/test_mllm.py
@@ -22,7 +22,7 @@ class FakeDataLoader:
}
def __iter__(self):
- for i in range(2):
+ for i in range(4):
yield self.data
@@ -63,8 +63,8 @@ class TestAutoRoundMLLM(unittest.TestCase):
bits, group_size = 4, 128
autoround = AutoRoundMLLM(
model, tokenizer, bits=bits, group_size=group_size,
- nsamples=1,
- batch_size=1, iters=2, dataset=self.dataset, quant_nontext_module=True,seqlen=256)
+ nsamples=5,
+ batch_size=3, iters=2, dataset=self.dataset, quant_nontext_module=False,seqlen=256)
autoround.quantize()
autoround.save_quantized("./saved/", format="auto_round", inplace=True)